diff --git a/clic/clusters/v3.2.0/README.md b/clic/clusters/v3.2.0/README.md new file mode 100644 index 0000000000000000000000000000000000000000..89a142569fe1535ef91328aec9f1567be27f99de --- /dev/null +++ b/clic/clusters/v3.2.0/README.md @@ -0,0 +1,47 @@ +# CLIC clusters: ParticleFlow v3.2.0 + +Attention and HEPTv2 checkpoints from the 28 October 2026 backbone campaign. +Both complete 50,000 optimizer steps with seed 12345 and global batch 128. +Training code: [`6b80b983477321cf708e5043fb1d2ec15dd398bc`](https://github.com/jpata/particleflow/commit/6b80b983477321cf708e5043fb1d2ec15dd398bc). +The v3.2.0 directory is a release label; this upload does not create a code tag. + +| Backbone | Original run | Parameters | Validation F1 | +| --- | --- | ---: | ---: | +| attention | [clic_pf_attention_seed12345_20261028_111202_052283](clic_pf_attention_seed12345_20261028_111202_052283/) | 3,114,258 | 0.8389 | +| heptv2 | [clic_pf_heptv2_seed12345_20261028_111202_849554](clic_pf_heptv2_seed12345_20261028_111202_849554/) | 8,639,954 | 0.8451 | + +## Files and checkpoint loading + +Each run preserves `checkpoints/checkpoint-50000.pth`, configs, the resolved +`particleflow_spec.yaml`, `model_kwargs.pkl`, scenario manifest, every training +log, complete available histories, separate `runs/train` and `runs/valid` +TensorBoard event streams, and `plots_step_50000`. The checkpoint contains +`model_state_dict`, optimizer state and `extra_state` with step 50000 and resume +state. Use the accompanying configuration with compatible ParticleFlow code. +The original full checkpoint bytes are retained. No best-checkpoint selection +or inference conversion is implied by the filename. + +[release.json](release.json) records dataset versions, model dimensions, +checkpoint hashes and metadata validation. [SHA256SUMS](SHA256SUMS) covers the +release contents except the checksum file itself. + +## Interpretation + +PF/clusters refers to the track/cluster-input MLPF model, not Pandora output. +Hit models reconstruct particle sets from raw hits. HEPTv2 and attention have +different parameter counts; hits and PF also differ in decoder, objectives and +target populations. All measurements are single-seed observations. + +Particle F1 uses capped matching cost and requires deltaR < 0.1 and relative +pT error < 0.5, without requiring PID agreement. PF uses TFDS 3.3.0 and hits +3.2.2. Validation uses 1530 PF entries or 1536 padded hit entries. Detailed +momentum uses a 512-entry subset with different selection across representations. +CLD hit HEPTv2 is missing its 30k history JSON; the final 50k result is present. + +[Comparison summaries](comparison_metrics/summary.json), +[metric definitions](comparison_metrics/provenance.json), and the archived +[study methodology](comparison_metrics/study_methodology.md) explain sampling +and timing caveats. Relative artifact links in that archived methodology refer +to the original local study; use the release links here for uploaded files. +Predictions, batch parquet dumps and dataset caches are not part of this release. +Training timing does not measure inference or upstream tracking/clustering cost. diff --git a/clic/clusters/v3.2.0/SHA256SUMS b/clic/clusters/v3.2.0/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..553b692e2feb9876b2502b1ffda82274a290464e --- /dev/null +++ b/clic/clusters/v3.2.0/SHA256SUMS @@ -0,0 +1,197 @@ +08987a2408529f6dd5cee49ab9b49f6ee14d54eab6923d6b8f01f11f80f6cf96 README.md +a87d438bbd6432178ad72c3e86e4031244fb6baf8c2044a2c3843fcc78a575a8 clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth +d7a10723cee9285e863b29a1b7b9582434ea730b8ae507d4724abea7e0fb8526 clic_pf_attention_seed12345_20261028_111202_052283/history/step_1000.json +b7a4880ef61628b9b77a2a301b42c0cce5b57ed6badffe264310514c70ec79dd clic_pf_attention_seed12345_20261028_111202_052283/history/step_10000.json +ad28f3e87cc90cc150fb79fbbddcd654c9b70a5eb9c1912039e9b285e51a5c2a clic_pf_attention_seed12345_20261028_111202_052283/history/step_11000.json +7255bd29be294dea78ff04349b00d650c3aff1e0395613708f932aff8c44ebca clic_pf_attention_seed12345_20261028_111202_052283/history/step_12000.json +00c62f1fcdb47965ef96a66868889d85dbedd5f2c1f4a01ec82297b43b3a4ffd clic_pf_attention_seed12345_20261028_111202_052283/history/step_13000.json +65283573143de9c58c891050818ff332455deeb332e5da2e7c259e5df12256e4 clic_pf_attention_seed12345_20261028_111202_052283/history/step_14000.json +097d2fa55ca14a0755df52fb552215ccc3bce2150d83300ab4d11d083146abed clic_pf_attention_seed12345_20261028_111202_052283/history/step_15000.json +2b4e09e47954df728cd36b94e161d575de349bd7303c220dc8565951eb048a52 clic_pf_attention_seed12345_20261028_111202_052283/history/step_16000.json +b8995944f9865854eac15edc1d8b8f35e9758d55b362bf29512190c7f77ac60b clic_pf_attention_seed12345_20261028_111202_052283/history/step_17000.json +69a93bf8534ded2eb8b8f8f10414bf564ef8807683a044d35d9b243fe63a7dc9 clic_pf_attention_seed12345_20261028_111202_052283/history/step_18000.json +4dfb37382a4c7f66198b2fa81518118c236a91c3788734193c38a65c5bdb5fbb clic_pf_attention_seed12345_20261028_111202_052283/history/step_19000.json +0f5be012d0bf52c21b2017143ef60444dd11e21a6e7087cad36d0d936640b1b5 clic_pf_attention_seed12345_20261028_111202_052283/history/step_2000.json +4c8be2322b039f2d797743ab83bf99d8073643cf21d1c0863765aed7f3e42439 clic_pf_attention_seed12345_20261028_111202_052283/history/step_20000.json +48db208c042337925fca94f4926d2d2b2337461e46bc673b876493bc96a16b3c clic_pf_attention_seed12345_20261028_111202_052283/history/step_21000.json +49900d01947e844308f935a1cde1028d9ca1e1b7ffb15070ca83b9b4a3c38418 clic_pf_attention_seed12345_20261028_111202_052283/history/step_22000.json +9c815d174fa40c537dd6575d315f88fbedaac653666d34ec669b86898bc7de82 clic_pf_attention_seed12345_20261028_111202_052283/history/step_23000.json +1ba4821f37a1c0a1744c3f9bd18460d119f5c526c4bb628c2041ed10cce0e5ec clic_pf_attention_seed12345_20261028_111202_052283/history/step_24000.json +1eccd7e5a7665ad12323524af0803c3e93629884806a777460fd8f5309bf2950 clic_pf_attention_seed12345_20261028_111202_052283/history/step_25000.json +72b90f95bc4553a254b5d95a1aed527c4f287cc4a4e1e3e1b01930f6d64884f8 clic_pf_attention_seed12345_20261028_111202_052283/history/step_26000.json +e83b74537891616805b790677a1742a54cce20c7e9a38e293d10e92115e779f7 clic_pf_attention_seed12345_20261028_111202_052283/history/step_27000.json +07d0ee50338c610e629753219313f4dda6020ac2e81f3faf1e5fb0eb051f0d61 clic_pf_attention_seed12345_20261028_111202_052283/history/step_28000.json +6bdd73eceb8e000fb8e625f74e675e91157cb2cd360ec25bf18c89d6097cd417 clic_pf_attention_seed12345_20261028_111202_052283/history/step_29000.json +f5683036fe2b56a88a6dd6d972bb9c9ebf9b78eb6973a486b39a0b50789e7c5f clic_pf_attention_seed12345_20261028_111202_052283/history/step_3000.json +b556f311c64a9efe9f8715f8ab4bd9798cedf35bf7cb36eca9b80a3c118c5828 clic_pf_attention_seed12345_20261028_111202_052283/history/step_30000.json +35133a455f88c9e2c146e580a2a83e49a35f9e7bb2fa7f34e7add2a513cc7ddb clic_pf_attention_seed12345_20261028_111202_052283/history/step_31000.json +d2a322dc8050f1cfd765d1b4018d51b6820bf182932564d8fca8a134f3da81fa clic_pf_attention_seed12345_20261028_111202_052283/history/step_32000.json +d5a4387e45628e8ce311a6bc6a0fefdd372c89eb65e6db811f3099c8fbf480d4 clic_pf_attention_seed12345_20261028_111202_052283/history/step_33000.json +1ce4efd098daf5be8be65005c052a52144c4c79ab7835ffbe42912bbaf6a9600 clic_pf_attention_seed12345_20261028_111202_052283/history/step_34000.json +0a2dabd637991bd53f396dd45b56c0e06143e904c055e86efb81c319a30e378e clic_pf_attention_seed12345_20261028_111202_052283/history/step_35000.json +9f2303a8dd40aeb5d25db40e4d3c1c34253c31cc08f50db579788060037538bb clic_pf_attention_seed12345_20261028_111202_052283/history/step_36000.json +b36c44cfe6bb1b4bb7690f93d305a198274051399dd501d19213bcf5b4bc20e7 clic_pf_attention_seed12345_20261028_111202_052283/history/step_37000.json +5475af444c655605d5333fbc4352eeaf153ead4b5764c51f23b36087f0d98f0c clic_pf_attention_seed12345_20261028_111202_052283/history/step_38000.json +d04853cc196ae2e20453d54832bbf4b1c6c1eda456e9db42a7ca1e6db4d33dc4 clic_pf_attention_seed12345_20261028_111202_052283/history/step_39000.json +ca67b82b427b36ce7e9c17fd7b4970ce406774d050b106abd64a34b8bd2b551b clic_pf_attention_seed12345_20261028_111202_052283/history/step_4000.json +d09eb6b788a7a40f7960810bdb59b5f439b7b2ca11a7c52ef26f7892de02cee2 clic_pf_attention_seed12345_20261028_111202_052283/history/step_40000.json +9b72ff29cb703382cc5ff72145fa6f71b0d21d0562b6ac1d2874e10bc08329a6 clic_pf_attention_seed12345_20261028_111202_052283/history/step_41000.json +e70a7a51436a0a600505d2f4bc459ca2d67bc95c4619a6c8535a537423014852 clic_pf_attention_seed12345_20261028_111202_052283/history/step_42000.json +e49930b4a3c995e0c341a04e27703d825e47bc35c69859826a777c532f7dc7d0 clic_pf_attention_seed12345_20261028_111202_052283/history/step_43000.json +57ab637359ec0e61860292f32e61cc067087e526225c596d23412fb1ee5eaa81 clic_pf_attention_seed12345_20261028_111202_052283/history/step_44000.json +39451b6acda0f263ad03e3a8b02c09b261623df9fd0ebab28ba708c84758f96c clic_pf_attention_seed12345_20261028_111202_052283/history/step_45000.json +5675432f2388857c2331a878454b1e09f4fc7c6f6624368f7a30caa3445c1fbf clic_pf_attention_seed12345_20261028_111202_052283/history/step_46000.json +72489078a1844dded376aba5e6553511b063cf9e6adfd665ac92e624a8b7cbc4 clic_pf_attention_seed12345_20261028_111202_052283/history/step_47000.json +bcf656d9c159ff10d563d20e90769c6858f65fa4420ff7c4d42be1e646403dca clic_pf_attention_seed12345_20261028_111202_052283/history/step_48000.json +33900afd8ee919e45ad0888f07148f5f90e556d1a0af7fbd6aace1a59289205b clic_pf_attention_seed12345_20261028_111202_052283/history/step_49000.json +f8f3275e32dc1db15710b8aebcf2e78db63a2a0ce896181b2dfe5943f3f691a1 clic_pf_attention_seed12345_20261028_111202_052283/history/step_5000.json +377cffa46588ea36e27a33e4b750f4876666ef62e3592fe3af6ec630c244ee72 clic_pf_attention_seed12345_20261028_111202_052283/history/step_50000.json +1c31afe552e364baefbd344d616244abceb3fe4b66346012e4ca66702a111591 clic_pf_attention_seed12345_20261028_111202_052283/history/step_6000.json +ec6ab9bb9f77b15713159fecf3da0c97ca3f120754bd7efd7d2ea327f74566af clic_pf_attention_seed12345_20261028_111202_052283/history/step_7000.json +aaf9c2f2e6f3321dd6a4f9d0536eac3125e884c02ff780bcc2001c772bf40549 clic_pf_attention_seed12345_20261028_111202_052283/history/step_8000.json +dc0f2889c5c86a201cc060966871b9180f3dbdd80e1aea1c03eeba87c39b7f6d clic_pf_attention_seed12345_20261028_111202_052283/history/step_9000.json +dc99b72054215fd4b01cd2e8bd1df37891c43cd646ae0933f150bd3fa15f22bd clic_pf_attention_seed12345_20261028_111202_052283/hyperparameters.json +1322be402e4e0cab113d6c1b0f7376da976a5e68bae3bf8cca3c8940d9176b96 clic_pf_attention_seed12345_20261028_111202_052283/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 clic_pf_attention_seed12345_20261028_111202_052283/particleflow_spec.yaml +5afa2394b00dd71ceb1af37ba511e044c93e1a5d1bab20f99a0556697b2f5f50 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf +021e5468856d3a510743c7f198e69801a3cc29d23344e366a4317f87815b1789 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.png +fda778d2e82276b43e941c740410c162109f60a1081ad92ab07626fd13d690c6 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf +58c65708235ab17531109db7a8f505f9de11a110d4216b563badc9cb8ebfb083 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.png +f2ce21105f2320610f41f37824005141d7155454e58468c0658752ecf315e604 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf +a41eec9b8480d931f6fa0cf3d3347502dc449bc4c073cabc9bd502f28557ce68 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png +f550b799a362576f07b678e56e7f9922a9dacc61459d0ae8ea7f45f4f184c8ca clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.pdf +4f2e711158cdb674f32216c38079f9fff93decce43a03f298858461c0eba0491 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.png +0667ba85c99934361b4e835355feba192ec5a4f2be7b73257b6eb0a25fc39e99 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf +a10ff67019780a866610b4fc862d8370ec644d6f0c92d8e3aef85562b28748a1 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png +c5bfaf80c0cba18f942820013d75bcb4b18cabfd86ccdc21228920fe7223103a clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.pdf +c3a81c0c6ddbbea95a62455804c800d81945e91a195d0026f018f93625e4f76f clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.png +7a5240c618c2212bcef88a2e8c739df3d89a10981166ff0f425b464fc159d22a clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf +092e164faf440dbf379a9baaaebc604b7621c55753dfb67af75048d81dcf6ac4 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png +d8d2d1cb35ab9f1a05d3e9a88aaf9a58754ef511b5a854516f4d3b46584cea66 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf +cfad398191f80ab4bf64c3c91e48ef3256e26388b02346583d127b5bc278ad1c clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png +7526095b59ce33de149c75236e30af61f877d2f6bad963883b33d0096eedf3d3 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf +6c63630f5bb9bb5a120bf3ed1b00b838be5a5af68613706b1b65dc1cb735d5be clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png +f94cd3d476af3476492053af36c305518492e4e4be4ab193f5df1ba0b6f26c00 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf +0c4921a5a1c6c465790e1ad5571c37b596ffddf994e23268c5d66651bd7935f2 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.png +c6768f0d047a44cedb2474901fdb8c1295b9ca8fe0c2d4e9326a684564ee7224 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf +d270f00f84f59b7f82bb913f836f498c0526b63438fcaa55b3e2328680462f67 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png +564375642f825490d6aec3a99ef256e331251965ed27299d5ac900282c42b5f4 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf +dbbca8ae4bcd8a25bb234b272dd6d03e19e23c1ab7f8cc3dded0001d8d1e4026 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.png +23155115b9d8a04aa555c71dbd308016d37a5fb0725ed4ecc0c3f623b437f307 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf +566baba35cfd1bf2f4e4a84a13d3b3032d4552ce658145aac50537e18c6bf263 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png +5511d5f01817b593594ead3f8872843c0bcbb62b1af83267407fb7d1352fa5d2 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf +fac439de516e74a31fe48aeccdb94ad5aeba1061d183efae8f638c86e4bb9062 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png +537c1f5f44c0f4069ffab672690ebd3dcf717fd13af51e0e65101c92e3727313 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf +12cf90798c854680185a835cbc2c94a4cbb4058634b1946edd7afc8683ffb703 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png +ed8c8a110557c4a125013ede3dbe5f98b23f48ca0866c301d53c1ac03c77a515 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf +e7d3d84664a7aded74897f7beb3add5694324bb77ddaca78955c152dda067fa8 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png +725e005dca0e95687bf1d2bb3af1493b5900bc1098c7a2444d4593afaaffa11c clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf +f6f7b7b36a70b2a6b5db1b5846cb2b5a1e281d7317c879bd2ab88bde69e653b5 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png +e181d0ef8119346a30ed9556e8a03d9064fc847900aac3c18ae8487412d927c9 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf +0e7d27506eb2d3c49a712ef015a23a7a279c3b895d5ea58a489b9691b3042a61 clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png +e7bb93355bb59b704baea3cc38e69f8286f1d3df6a68abcd203b91b3792283db clic_pf_attention_seed12345_20261028_111202_052283/runs/train/events.out.tfevents.1793178753.nid005777.42008.0 +38a7ce1242092f1babbb18646211c263f6e0ce05636187fac9b4911421de8fdc clic_pf_attention_seed12345_20261028_111202_052283/runs/valid/events.out.tfevents.1793178753.nid005777.42008.1 +a4636c53bea4e202aa2f9ca0865245925a1da4c012a5a79eb9e5ff099ed23503 clic_pf_attention_seed12345_20261028_111202_052283/scenario-manifest.json +b2e54f76073ebf0710949ddfa333afe3a12f3095051ec58ff5eef3543926ee29 clic_pf_attention_seed12345_20261028_111202_052283/train-config.yaml +9b5c2ba03ed0e9357e2423bcdf0f12508a8ae848476899989961c6e6b58ef06c clic_pf_attention_seed12345_20261028_111202_052283/train.log +ba41c41e45083916af49b7f179a14354095eae20780ac04e07b40e01fa85f8f6 clic_pf_attention_seed12345_20261028_111202_052283/train.log.0 +4ad2caa8fcc5adf4a316b44231e6bddbf83606b701a566b8f3f57876b756ab2e clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth +274dcfbb303fdfab79bb7b96254e6430da98301cee3781d9f712a5cc70f018b0 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_1000.json +bb0494a1582723813c38b48a50d4f92bf9c0b1f089c2386fdd667ea373133c51 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_10000.json +549e0773a0a1c885626d09dd4e7564ff0e9d0c9aa9642d499b7e91cf92df1fa6 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_11000.json +423caa0075850fc16c38fe545dc06b1a2e8c796128731caaa69e26495ea3f7cb clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_12000.json +e85a50d9402917e928cc59dc4e3505e06da158c0aabac4583af11c36d1d9a0fe clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_13000.json +36460254c20fd5ca4590b71b8c9124893b30872dec5fc7e8f05ad42e1495e0c6 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_14000.json +065fdd9a6e7d8ef9bf0246935eeac8de31a4bdc9dd00798152eaea2684cf1aa6 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_15000.json +0ad49ae15afc9e055bae82c03e74b85e17fe3d70352debe0392ef382bf37b687 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_16000.json +da1bc74405e49ad78229338c78680532189555a8476f46a9ccfb5d8ec91098f3 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_17000.json +9bb2585df0826cc5a1df00c0c2079ef3ad9dfa493b4fa01ee2ed3d0bbac6fd59 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_18000.json +ce4d82658b69ed7b2af6bdb6ae1fd3941a5612287c501658bc6f1abcd1af72fe clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_19000.json +1781dc0e78ab6e427cb82477724085e284b99f2bae3dff0efb5bbd62c4f4a6e8 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_2000.json +babfdac4794b99a82f92d1b285bbff3f0ef9d2215ce970b93dd3ca173bb1a4db clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_20000.json +6760f2ff9445149046b83792ad31dc2eba9bdc5455d2e3fad86e48eb8d2e7607 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_21000.json +838ab2b4fe5c48192908992ae769450daa48491eaae81c84d803abaab638d3a7 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_22000.json +71843b3d542dd93036357a05b3c5b3c2bd67a5acf55fe42575bb38c6e6fbacc8 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_23000.json +2ee6292232dc1ac493668971b3b7b4d76086ff2b03c5ee22bf4787d4e51dfd37 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_24000.json +2e78f11a6418730611233e0e2e5d77e5b5ca637cb73c36dc3c1764fea1519fcc clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_25000.json +5644d10f44f5deccabeeb3e0d24469942ef388384715b7b3551e9e5133580c66 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_26000.json +dbdf50089effd8aed58cdf385ad6b1f6ce08ad93604b9496be8732f5c6f6832e clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_27000.json +c32aedcff7c0e617940a9c7e3fa1713788ffe306f8ccf5a0e43ac0b27055ad59 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_28000.json +05c9354d57d8dc101662c553a4550e310c04f794ac2fa6994bcbdc4dc69a988a clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_29000.json +e2ebf8973cc656d9748ad568789d527223839ac521ce541c827ff1af83c767c1 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_3000.json +36e00ab933eeb6f4d785cedcf7bff2fbe99dfd98edf1153d9c3bfe66c1146c61 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_30000.json +514196c31248baf511c6a51e4cb4291b7a9de5f66ca9fef6b0efcfa9f9fd091d clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_31000.json +fbfeb4b5006024021c8df4de8d0e1a987a02f915c675309e726310dd32ec39bb clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_32000.json +e421d451c4898feeff203281f39b87b0b58ef89b57ea6c940d51c963a8f7e4b6 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_33000.json +a51d822acb0dc9d849fcc3d5045fed5eb3afa035252bf480a00f34f34e8aba74 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_34000.json +f736a5e06f9f3c2276b610a1fa85e0c3a59614b94159eefe92f31ca3b3bbe63e clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_35000.json +7e9899894ab59fb69ef4ccc76ba5716953eff6c798f6aa24389b8e82eb2e161f clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_36000.json +5d680c9ece1c528450a39a79ac35825a91573b5809cd6a96394afeb5a410b9db clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_37000.json +17045da6224677f78fb8d50bc08b73a8082c4cf8fcedaca0c6da74af078499f5 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_38000.json +d23dd9b61e69b051598d4b55faeca1dbae5b12465c0282a58dd8d218c72e41b7 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_39000.json +63f189b759a3b65204017a947524cae8756c2b8d7e7d8c42cc1bbfc3ba3b92f3 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_4000.json +47d4249b223e03032165cf9320106258a806542a4f1fb6e99cdb30b719d8bebd clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_40000.json +7d5ebd9dd50618ba642ded41cafec9f17d41fd51a365e0acb0c7667ba3c01606 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_41000.json +6ec1546fc90274cb6d935c7fe8a61cf81d8ada9c088649bdc798c7b5ab1af0f3 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_42000.json +ec506ffd9918ea31d35491093000227ad41a55e0cf5ab9e016c8d5551f6003a1 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_43000.json +3a1023ed7856664447df952483d3ff41e0c3e6a68201dc70b48ba8e56af3c62e clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_44000.json +ab68be80f856a4eae34bd356a5294d629348d2ce645e204db9172cd31fbc7024 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_45000.json +1f546f00c97d73524b17136f275f276f6ff9bd37aa86f749769e46b5cabee6b6 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_46000.json +7f6455aff4556b5f03957a22d6b355d3a4d97f21ddaf438716f505ce3ee879d2 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_47000.json +3783660f1366e11464107b8ec39e50503e1ce4f9733995082bb4481ac68b9cbc clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_48000.json +1d0b161268e775a6ed614f1c60978cb587a1ac362f3d11f3824a808fb6ccad60 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_49000.json +5ce6018a0c76c1a7725498086f8c21f61dc0ab35f3304cfb1c6d2b9cb39b5179 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_5000.json +3815be5804165c27b4a0ecf0fba97ec4b0dabc14e1f9b4674194ec1618d2d3dd clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_50000.json +f14c29c74589cafb7e8f032be475647a54c9a7e8fe718b270230eaa440d3b5e7 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_6000.json +6b00290a288989ac85de1b3e8a40819cc363abc0ef17655a4cfb29cfdf70e58d clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_7000.json +f16cee3569762fc22a90a0f179fc5d1731a9c19a12aa1579c975771547d6d39f clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_8000.json +6a7595f7c16760057ea83b397df00db3fd774bf2f42be391825c2037930ed7d5 clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_9000.json +1ab02c82982efddc2810f16f34ac4d2f1e965d88805c7e839b641c857525d86e clic_pf_heptv2_seed12345_20261028_111202_849554/hyperparameters.json +1ca7e4d3eb5002a66d9e0314201360d711a541e8b601ce0ec81f5e8f7c3e6da2 clic_pf_heptv2_seed12345_20261028_111202_849554/model_kwargs.pkl +0b760998f19c1189d9d765e69de13a138fa6c0aa679cda57a5cba137f70ddff1 clic_pf_heptv2_seed12345_20261028_111202_849554/particleflow_spec.yaml +5da0a1d70a32aeed8b7a701954cdd33feba773bae2728065080300e213bc35c0 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf +c01e483edd06da01a62a358936bec9b9ba1dc86accd944c8a8f02191ad1a213a clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.png +0b9f9a47e5b84b606970fe48bc47861908fc3d682f3fd2aaef9e836a6c1bd341 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf +073077cd32c14565788cc7c21da09168be0296191fe84f23d4feaf5430dec0b5 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.png +3d55812c84221cbc9b51a10c8a0696ee1b7cca09295857faf09eb18e9e52470c clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf +937895313c69a0a4c0b0b65517b2dff0011b32075f3e549dc41e769afe1029ca clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png +33344dbcd3cefd1981f13fe87c097be45cae98cdc358f177a1946c265ff6ae3d clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.pdf +6eb91613fc006bd4b4fa00a221addfa10d3c0a603550f5b0cc1b613f78006326 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.png +7969151d532c91fbf7e11c3c06bf85bb251b1f46f2e432133ad8e044e0f23115 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf +3492f44e4da8ff17cbf18f03755ae1e9ee6131eecbbac15bdb04fe51e84b4761 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png +f3619cb871d6aadaf4cf87b8a8ea9ac3afbf921361a9cf7a53cb25101d26006b clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.pdf +c3a81c0c6ddbbea95a62455804c800d81945e91a195d0026f018f93625e4f76f clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.png +cfae28d10e4aa406204b1308eef911ec04195619f3231b68b71091c70d7ff960 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf +c33c3e1facf1431450cd49859d55dae14042d3a709460d20a8ef44521421d617 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png +d3b7414da4ca193509b913454a60ff50d03b26f54ad888b57d548f7998ee70bf clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf +88f48bcdb7231ed2223d3dec005a341f2bdc870d248dcc00fe5c4f96f421866c clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png +3e735e36445fcb6b46311f757ffd32ffd85831715223ff16e1e25058d14bccb6 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf +89a68d99064d3fced76316bc78350a6cc96c5b175233e6ef0bad666fc729b3bc clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png +f71ef1edb02eef33e470001fee7c37a25b1ed164fb388c7ca8ad5ef7e42746c0 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf +e3f3b3dc00ba1834a01095be1d162d1fe3d263fffff123393bc655f0a6713862 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.png +7fd0cfee4bcd97fd50d47e210cb9a94bdb55c00b7b7553f02095cc9b134de162 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf +601a0ca058547f81e3216ab3773b2743ec84317c26a1f218f8aa7996a4378d58 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png +a921ab3a07e1988123ffb6dbfb3a17d8c5fd1e3dd91ccc95ca663a5bfb54d1d1 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf +dbbca8ae4bcd8a25bb234b272dd6d03e19e23c1ab7f8cc3dded0001d8d1e4026 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.png +0ca83fbe5dc0aa594026e7ca4bc00b076a8716661c582a04214461f40efa5375 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf +1bdab69254769ee212b83d803f597c5d9456e044a0de1957e7d0ee05a0a839e1 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png +4af4b3bd332e531c7ec4a33d8ff050f5b7381a0e3dc7edaf3ff2fb122878ca03 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf +22cb5cdebd94718c3effd797d8d0ad06278caecff3d6dd566e1342952755aa37 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png +23f0281bc041751eedd8a71f16cef7a97a086d3a5eb809629723cc9e82ea4071 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf +f1452358d1de8b31947a845cc26390baeb9051f88d9a69a247dc5c6d76f6d093 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png +a0c039ed2c684d036f37067c2561d0be4189b14139a5f21d6fb997e038a9c078 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf +d1ec7b554e07caba5b0abb0739a012502d6354a8fdc2a5b71b3a8dd4edd503b6 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png +35f1e04d90a383dcdec79c84b7d725d2f74ef0339ce57d8e2183d427fb772585 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf +0fbbde5d7fe1adfc20a526a954f7c64206cf1af0965dd92bf1b65aa83abab0f6 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png +6d47f12279d0f7c6006bfba16c8a9191fedff652b770c989c5451ab141a0518f clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf +0e7d27506eb2d3c49a712ef015a23a7a279c3b895d5ea58a489b9691b3042a61 clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png +4dc2a84f86e72c8311be083c54ec242a8adbe0bc6d2ed3f977bdb53d27a96188 clic_pf_heptv2_seed12345_20261028_111202_849554/runs/train/events.out.tfevents.1793178753.nid005803.1218.0 +3f8fa0ea6cd853d9eb82f129f91e1a70a6c2257d3b324549db3f7358a4b48ad7 clic_pf_heptv2_seed12345_20261028_111202_849554/runs/valid/events.out.tfevents.1793178753.nid005803.1218.1 +23e6919aa4a24d2c8f3e8b80339f723e77496543c8d7adc8587281f667f51b10 clic_pf_heptv2_seed12345_20261028_111202_849554/scenario-manifest.json +019c1e01633332aebe8c77f2c0cfe8a3ef6c256087aa3b37d4442c3ce1c997d0 clic_pf_heptv2_seed12345_20261028_111202_849554/train-config.yaml +b45537068703fbd91895e59da9e3650217d8924584a84e858a5caaee3ff9838e clic_pf_heptv2_seed12345_20261028_111202_849554/train.log +65e7874a6f928e70b152a8ec1c03690c458101c3995f47d81459d0f4d521b6fc clic_pf_heptv2_seed12345_20261028_111202_849554/train.log.0 +a9a813daf88031a86204a1100e12ecad2caf059954dd8256d7d98e46b2171105 comparison_metrics/provenance.json +4b00f93f03bf042116ccedcd743f4f78b0d26cc18501fca9bc294975f2e4beb1 comparison_metrics/study_methodology.md +d095da9102d691ca981024ae79adf73e4ed098cf411ad4f88d6de49d99a5963a comparison_metrics/summary.json +e84eac1ede31d6f6517490aca03d4c49aab8cabd9944313bcd4cf90447ce0390 release.json diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..20efa82f98d048905a67a8bc2029daf205da30a0 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87d438bbd6432178ad72c3e86e4031244fb6baf8c2044a2c3843fcc78a575a8 +size 37511559 diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_1000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..541a5ee9a1d0524b438f6e845f7de469986041f9 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.04737540269092287, "train": {"Classification_binary": 4.275143623352051, "Classification": 0.03892771154642105, "Regression_pt": 0.21984045207500458, "Regression_eta": 0.016466049477458, "Regression_sin_phi": 0.012514754198491573, "Regression_cos_phi": 0.013042405247688293, "Regression_energy": 0.21934209764003754, "Total": 14.155025482177734, "Total_unweighted": 4.795277093537152}, "train_group": {}, "valid": {"Classification_binary": 4.1719512939453125, "Classification": 0.04046350469191869, "Regression_pt": 0.24245818456013998, "Regression_eta": 0.01638021692633629, "Regression_sin_phi": 0.013093164811531702, "Regression_cos_phi": 0.01343549539645513, "Regression_energy": 0.24481709798177084, "Total": 14.841283162434896, "Total_unweighted": 4.742598958313465, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.042137131094932556, "diagnostic/loss/clic_pf/Classification_binary": 4.19295072555542, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.013191940262913704, "diagnostic/loss/clic_pf/Regression_energy": 0.25096914172172546, "diagnostic/loss/clic_pf/Regression_eta": 0.015154458582401276, "diagnostic/loss/clic_pf/Regression_pt": 0.24920544028282166, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.013009442947804928, "diagnostic/regression/clic_pf/energy_pred_mean": 0.046751849353313446, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2533818781375885, "diagnostic/regression/clic_pf/energy_residual_mean": 0.062305472791194916, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5624197125434875, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.026618430390954018, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2572077810764313, "diagnostic/regression/clic_pf/pt_residual_mean": 0.09546297043561935, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5704606771469116, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9514381885528564, "metrics/particle/class_1/pid_accuracy": 0.9862109422683716, "metrics/particle/class_2/efficiency": 0.17749211192131042, "metrics/particle/class_2/pid_accuracy": 0.14612452685832977, "metrics/particle/class_3/efficiency": 0.6344002485275269, "metrics/particle/class_3/pid_accuracy": 0.9808717966079712, "metrics/particle/class_4/efficiency": 0.9467054009437561, "metrics/particle/class_4/pid_accuracy": 0.008188331499695778, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.0020408162381500006, "metrics/particle/count/bias_mean": 2.0640523433685303, "metrics/particle/count/mae": 4.881045818328857, "metrics/particle/count/prediction_mean": 72.06340026855469, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.130199134349823, "metrics/particle/event/energy_response_mean": 1.0326772928237915, "metrics/particle/event/met_abs_error": 16.52463150024414, "metrics/particle/event/scalar_pt_relative_abs_error": 0.11652738600969315, "metrics/particle/event/scalar_pt_response_mean": 1.0433753728866577, "metrics/particle/event/vector_pt_closure": 0.12207666784524918, "metrics/particle/matched/delta_eta_abs_mean": 0.020847324281930923, "metrics/particle/matched/delta_phi_abs_mean": 0.014793637208640575, "metrics/particle/matched/delta_r_mean": 0.02860811911523342, "metrics/particle/matched/energy_relative_abs_mean": 0.08996055275201797, "metrics/particle/matched/pid_accuracy": 0.9498991966247559, "metrics/particle/matched/pt_relative_abs_mean": 0.08811218291521072, "metrics/particle/matching/duplicate_fraction": 0.039480485022068024, "metrics/particle/matching/efficiency": 0.7501283884048462, "metrics/particle/matching/f1": 0.739229679107666, "metrics/particle/matching/purity": 0.7286430597305298, "metrics/particle/matching/target_coverage_dr0p05": 0.646990180015564, "metrics/particle/matching/target_coverage_dr0p10": 0.7890269756317139, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6590537428855896, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8074305057525635, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.03417092189192772, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08689515292644501, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08792351931333542, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024054640904068947, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14446234703063965, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003781228791922331, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3889233469963074, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.047235988080501556, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2914783954620361, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7405.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.5107855200767517, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 2.0613834857940674, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.094794511795044, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.941170334815979, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.21490977704524994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.684106826782227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.029819563031196594, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.3430522680282593, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.05611415207386017, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.198084354400635, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10530.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6889013051986694, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.612678050994873, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 16.898530960083008, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 7.527128219604492, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.7685967683792114, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5639433860778809, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 14.677337646484375, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -3.999168634414673, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 24.516132354736328, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.041869424283504486, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10252422839403152, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09782694280147552, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03209996223449707, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15733912587165833, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.015094339847564697, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9815530776977539, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.048868488520383835, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9185359477996826, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1325.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.2271381616592407, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.1100003719329834, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 122630.8125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 122630.6015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.4057970941066742, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 6601298.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.11732228845357895, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.957142353057861, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.39735686779022217, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 15.993682861328125, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2898.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4313150942325592, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 3.1341781616210938, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 15.463818550109863, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.5481297969818115, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11351979523897171, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0957883894443512, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 15.103769302368164, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3802181482315063, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 33.7796745300293, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025495262816548347, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0903870239853859, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0874880775809288, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02959355339407921, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1412588506937027, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008283174247480929, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12671247124671936, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006688878871500492, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6224910616874695, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18109.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2384554147720337, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1754156351089478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.839289665222168, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7024322152137756, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12762027978897095, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.233697891235352, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0070181917399168015, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.5041348338127136, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010368558578193188, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 10.643444061279297, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21658.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8248610496520996, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.82086443901062, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 22.582075119018555, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 18.5689754486084, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.009536027908325, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8392221927642822, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 11.018661499023438, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.5059963464736938, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 68.9435806274414, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.028697283938527107, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09002279490232468, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08811862766742706, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.028189077973365784, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14298129081726074, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002347330329939723, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24125978350639343, "metrics/particle/momentum/all/accepted/pz_bias": 0.010932015255093575, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.068715214729309, "metrics/particle/momentum/all/accepted/samples": 26839.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.40184932947158813, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.6011031866073608, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 10130.08984375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 10129.94140625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1767941564321518, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1897193.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.022972125560045242, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.1237157583236694, "metrics/particle/momentum/all/assigned/pz_bias": -0.04326113313436508, "metrics/particle/momentum/all/assigned/pz_residual_rms": 9.958165168762207, "metrics/particle/momentum/all/assigned/samples": 35086.0, "metrics/particle/momentum/all/efficiency": 0.7502166032791138, "metrics/particle/momentum/all/event/energy_bias": 10.567719459533691, "metrics/particle/momentum/all/event/p3_error_mean": 42.851863861083984, "metrics/particle/momentum/all/event/pt_vector_error_mean": 22.669193267822266, "metrics/particle/momentum/all/event/px_bias": -1.1274192333221436, "metrics/particle/momentum/all/event/py_bias": -1.179490566253662, "metrics/particle/momentum/all/event/pz_abs_mean": 31.61168670654297, "metrics/particle/momentum/all/event/pz_bias": -3.8733904361724854, "metrics/particle/momentum/all/event/pz_residual_rms": 80.37352752685547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.020433783531188965, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04326183721423149, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03878562152385712, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.02746368758380413, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07554230839014053, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014351678546518087, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15585796535015106, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.023222383111715317, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8575670123100281, "metrics/particle/momentum/class_1/accepted/samples": 16026.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06971130520105362, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.15634498000144958, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.20304983854293823, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17609204351902008, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.031470395624637604, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.100799083709717, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.007941681891679764, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.4637988805770874, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.10717317461967468, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 12.043587684631348, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9498014450073242, "metrics/particle/momentum/class_1/event/energy_bias": 27.80210304260254, "metrics/particle/momentum/class_1/event/p3_error_mean": 26.060379028320312, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 15.058940887451172, "metrics/particle/momentum/class_1/event/px_bias": -2.4777143001556396, "metrics/particle/momentum/class_1/event/py_bias": -2.1522982120513916, "metrics/particle/momentum/class_1/event/pz_abs_mean": 18.298940658569336, "metrics/particle/momentum/class_1/event/pz_bias": 4.007892608642578, "metrics/particle/momentum/class_1/event/pz_residual_rms": 72.27355194091797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.051762860268354416, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23823648691177368, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23400317132472992, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07031131535768509, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.27367228269577026, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.046210721135139465, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9699420928955078, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.06718610972166061, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.272760391235352, "metrics/particle/momentum/class_2/accepted/samples": 541.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.6025214195251465, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.3127745389938354, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 134001.71875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 134000.9375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.592383086681366, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6900678.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.1493212729692459, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 5.57403039932251, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.3933849632740021, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 13.834583282470703, "metrics/particle/momentum/class_2/assigned/samples": 2652.0, "metrics/particle/momentum/class_2/efficiency": 0.1836388260126114, "metrics/particle/momentum/class_2/event/energy_bias": -27.49773597717285, "metrics/particle/momentum/class_2/event/p3_error_mean": 29.135461807250977, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.213557243347168, "metrics/particle/momentum/class_2/event/px_bias": 0.7120314836502075, "metrics/particle/momentum/class_2/event/py_bias": -0.2024010270833969, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.967945098876953, "metrics/particle/momentum/class_2/event/pz_bias": -3.7630317211151123, "metrics/particle/momentum/class_2/event/pz_residual_rms": 34.125675201416016, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.04142576456069946, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16080747544765472, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16310028731822968, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03566775843501091, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2055734395980835, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013333333190530539, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2804197072982788, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01390144694596529, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9188372492790222, "metrics/particle/momentum/class_3/accepted/samples": 9750.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5758332014083862, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.3291146755218506, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.213972806930542, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.0377392768859863, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.27202504873275757, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.801692962646484, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0181793961673975, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.103094220161438, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1491805613040924, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.887613296508789, "metrics/particle/momentum/class_3/assigned/samples": 15017.0, "metrics/particle/momentum/class_3/efficiency": 0.6326239109039307, "metrics/particle/momentum/class_3/event/energy_bias": 22.511558532714844, "metrics/particle/momentum/class_3/event/p3_error_mean": 21.817420959472656, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 11.100020408630371, "metrics/particle/momentum/class_3/event/px_bias": 0.3385649025440216, "metrics/particle/momentum/class_3/event/py_bias": 0.7360332012176514, "metrics/particle/momentum/class_3/event/pz_abs_mean": 16.454708099365234, "metrics/particle/momentum/class_3/event/pz_bias": -3.7091050148010254, "metrics/particle/momentum/class_3/event/pz_residual_rms": 28.212411880493164, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.023729929700493813, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.06378549337387085, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06444226205348969, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.016306951642036438, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11026620864868164, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.005797101650387049, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3474128544330597, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05951828509569168, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.3050122261047363, "metrics/particle/momentum/class_4/accepted/samples": 345.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.059727467596530914, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.13545380532741547, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.15801776945590973, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.10846229642629623, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03038674034178257, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 0.9161726832389832, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005524862091988325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3807588815689087, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.021371951326727867, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3591943979263306, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9530386924743652, "metrics/particle/momentum/class_4/event/energy_bias": -5.746252059936523, "metrics/particle/momentum/class_4/event/p3_error_mean": 6.197022438049316, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.838681221008301, "metrics/particle/momentum/class_4/event/px_bias": 0.5492201447486877, "metrics/particle/momentum/class_4/event/py_bias": 0.22229896485805511, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.9690895080566406, "metrics/particle/momentum/class_4/event/pz_bias": 0.2131918966770172, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.201930046081543, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.014931422658264637, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022840097546577454, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02475336566567421, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006132765673100948, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05204405635595322, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.32600992918014526, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0005647051730193198, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9872056841850281, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.023363905027508736, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07947567850351334, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0813232809305191, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.05247504264116287, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.47474658489227295, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6356331706047058, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.27707940340042114, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.134149074554443, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -6.501951217651367, "metrics/particle/momentum/class_5/event/p3_error_mean": 6.062408447265625, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 4.751188278198242, "metrics/particle/momentum/class_5/event/px_bias": -0.24952149391174316, "metrics/particle/momentum/class_5/event/py_bias": 0.21687650680541992, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.978724479675293, "metrics/particle/momentum/class_5/event/pz_bias": -0.6223385334014893, "metrics/particle/momentum/class_5/event/pz_residual_rms": 9.46342658996582, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.117197036743164, "step_valid_time": 250.01645946502686, "step_total_time": 251.13365721702576, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0102811796112552, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19395268966103285, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2715, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2504, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.900497512437811, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8951533135509396, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8978174603174603, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10484668644906034, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.830514096185738, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.82558522914606, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.828042328042328, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.900497512437811, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0120098063028133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21334211693670802, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1941, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1848, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8615179760319573, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8645879732739421, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8630502445531347, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1354120267260579, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8202396804260985, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.823162583518931, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8216985326811916, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8615179760319573, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.016147194687731, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.233797817621763, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1293, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1014, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8342412451361868, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8290796597061099, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8316524437548487, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.17092034029389017, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7891050583657587, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7842227378190255, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7866563227307991, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8342412451361868} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_10000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..ac5cfabc38b07cb1626c3af2a1e949a3f45ccef2 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47375402690922874, "train": {"Classification_binary": 3.196850061416626, "Classification": 0.028186507523059845, "Regression_pt": 0.19974495470523834, "Regression_eta": 0.016383858397603035, "Regression_sin_phi": 0.013445882126688957, "Regression_cos_phi": 0.011621505953371525, "Regression_energy": 0.19764801859855652, "Total": 12.11905288696289, "Total_unweighted": 3.663880788721144}, "train_group": {}, "valid": {"Classification_binary": 3.0918375651041665, "Classification": 0.025618751843770344, "Regression_pt": 0.15969425439834595, "Regression_eta": 0.012541075547536215, "Regression_sin_phi": 0.0096922367811203, "Regression_cos_phi": 0.00968500847617785, "Regression_energy": 0.16246907909711203, "Total": 10.304018020629883, "Total_unweighted": 3.471537971248229, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.026195846498012543, "diagnostic/loss/clic_pf/Classification_binary": 3.0857930183410645, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009333960711956024, "diagnostic/loss/clic_pf/Regression_energy": 0.16475652158260345, "diagnostic/loss/clic_pf/Regression_eta": 0.01177170593291521, "diagnostic/loss/clic_pf/Regression_pt": 0.16252288222312927, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009386595338582993, "diagnostic/regression/clic_pf/energy_pred_mean": 0.062330130487680435, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.21318629384040833, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0778837502002716, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4723050594329834, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.017788926139473915, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20947769284248352, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08663345873355865, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4786947965621948, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9589917659759521, "metrics/particle/class_1/pid_accuracy": 0.9868377447128296, "metrics/particle/class_2/efficiency": 0.37189894914627075, "metrics/particle/class_2/pid_accuracy": 0.652213454246521, "metrics/particle/class_3/efficiency": 0.7070804834365845, "metrics/particle/class_3/pid_accuracy": 0.9806432127952576, "metrics/particle/class_4/efficiency": 0.932170569896698, "metrics/particle/class_4/pid_accuracy": 0.25467774271965027, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.48163264989852905, "metrics/particle/count/bias_mean": 0.5888888835906982, "metrics/particle/count/mae": 3.7104575634002686, "metrics/particle/count/prediction_mean": 70.5882339477539, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0715973973274231, "metrics/particle/event/energy_response_mean": 1.0407190322875977, "metrics/particle/event/met_abs_error": 6.484316825866699, "metrics/particle/event/scalar_pt_relative_abs_error": 0.057966575026512146, "metrics/particle/event/scalar_pt_response_mean": 1.037899374961853, "metrics/particle/event/vector_pt_closure": 0.05544101074337959, "metrics/particle/matched/delta_eta_abs_mean": 0.01567261293530464, "metrics/particle/matched/delta_phi_abs_mean": 0.012576661072671413, "metrics/particle/matched/delta_r_mean": 0.02256379835307598, "metrics/particle/matched/energy_relative_abs_mean": 0.08956743776798248, "metrics/particle/matched/pid_accuracy": 0.9605123400688171, "metrics/particle/matched/pt_relative_abs_mean": 0.08030915260314941, "metrics/particle/matching/duplicate_fraction": 0.03587963059544563, "metrics/particle/matching/efficiency": 0.8011186122894287, "metrics/particle/matching/f1": 0.7977628707885742, "metrics/particle/matching/purity": 0.7944352030754089, "metrics/particle/matching/target_coverage_dr0p05": 0.7402403354644775, "metrics/particle/matching/target_coverage_dr0p10": 0.8401665687561035, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7047404646873474, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8455167412757874, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024350976571440697, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.088286392390728, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08122365176677704, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021760964766144753, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13885845243930817, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00376991368830204, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3849303424358368, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.007125940639525652, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.321492314338684, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8223.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.346269428730011, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2720509767532349, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3488022089004517, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2228963375091553, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15284553170204163, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.671087265014648, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018842659890651703, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9334977269172668, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.025458810850977898, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.118218183517456, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10455.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7650014162063599, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.241119384765625, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.509072303771973, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.774716377258301, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.17265093326568604, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07443919777870178, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.301374912261963, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.5233027935028076, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.369612693786621, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.032446280121803284, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10389070212841034, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09733715653419495, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.027185935527086258, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1563635617494583, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018529588356614113, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0002961158752441, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.16147862374782562, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2155351638793945, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1673.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.870841383934021, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8615667819976807, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.449115753173828, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.2753076553344727, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2920757234096527, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 36.404720306396484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08660589158535004, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.5954785346984863, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11436410993337631, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.079439163208008, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2852.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5445963740348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.9301706552505493, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.063070297241211, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4840599298477173, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06262840330600739, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.18186001479625702, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.833060264587402, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.8813115358352661, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.67518424987793, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021021325141191483, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09005054831504822, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07923980057239532, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02369813621044159, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13678406178951263, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008541077258996665, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1372247338294983, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.05149632692337036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6288986206054688, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18733.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18123376369476318, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8091011643409729, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6123207807540894, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.505579948425293, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10725989192724228, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.312222480773926, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0035999445244669914, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3197050392627716, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.06041776388883591, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.58798086643219, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21667.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8532841205596924, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 5.651818752288818, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.247167587280273, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.5306854248046875, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.8644869327545166, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7636638879776001, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.693875312805176, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.6980750560760498, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.312469482421875, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02264533005654812, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0903526172041893, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08086717873811722, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023345550522208214, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13859842717647552, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027245101518929005, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25880780816078186, "metrics/particle/momentum/all/accepted/pz_bias": -0.026306316256523132, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1681677103042603, "metrics/particle/momentum/all/accepted/samples": 28629.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2868039011955261, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0333184003829956, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0638118982315063, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9458728432655334, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13595813512802124, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.753555297851562, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014925372786819935, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7703171372413635, "metrics/particle/momentum/all/assigned/pz_bias": -0.03571444749832153, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.045679569244385, "metrics/particle/momentum/all/assigned/samples": 34974.0, "metrics/particle/momentum/all/efficiency": 0.8002515435218811, "metrics/particle/momentum/all/event/energy_bias": 10.823108673095703, "metrics/particle/momentum/all/event/p3_error_mean": 19.823204040527344, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.31847858428955, "metrics/particle/momentum/all/event/px_bias": -2.099766254425049, "metrics/particle/momentum/all/event/py_bias": 1.019963026046753, "metrics/particle/momentum/all/event/pz_abs_mean": 15.310003280639648, "metrics/particle/momentum/all/event/pz_bias": -3.340066432952881, "metrics/particle/momentum/all/event/pz_residual_rms": 23.574169158935547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01341115403920412, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04247657209634781, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.026245804503560066, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009720036759972572, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06779838353395462, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011140682036057115, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14545802772045135, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.030813438817858696, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7719072699546814, "metrics/particle/momentum/class_1/accepted/samples": 16157.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06229515001177788, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23208226263523102, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.21786169707775116, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.18787692487239838, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02744028903543949, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.089659690856934, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005037634167820215, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31098106503486633, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.012454364448785782, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.468390941619873, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9575653672218323, "metrics/particle/momentum/class_1/event/energy_bias": 9.072769165039062, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.655338287353516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.6087422370910645, "metrics/particle/momentum/class_1/event/px_bias": -1.3333848714828491, "metrics/particle/momentum/class_1/event/py_bias": -0.14379896223545074, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.334673881530762, "metrics/particle/momentum/class_1/event/pz_bias": -0.7459398508071899, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.828893661499023, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.044013138860464096, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20784588158130646, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2060765027999878, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04868195578455925, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24599267542362213, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04098360612988472, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8444348573684692, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03994520753622055, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9972283840179443, "metrics/particle/momentum/class_2/accepted/samples": 1098.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0544565916061401, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8991473913192749, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.1018764972686768, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5599115490913391, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4254451096057892, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.523874282836914, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09384272992610931, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.761259078979492, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2423417568206787, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.850797653198242, "metrics/particle/momentum/class_2/assigned/samples": 2696.0, "metrics/particle/momentum/class_2/efficiency": 0.3727087676525116, "metrics/particle/momentum/class_2/event/energy_bias": -6.280264854431152, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.71731948852539, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.63237190246582, "metrics/particle/momentum/class_2/event/px_bias": -0.6276596784591675, "metrics/particle/momentum/class_2/event/py_bias": 0.2976670265197754, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.467366218566895, "metrics/particle/momentum/class_2/event/pz_bias": -1.7868812084197998, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.035314559936523, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03455081209540367, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1500275880098343, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15112358331680298, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05148722976446152, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19316346943378448, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012893718667328358, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2614639699459076, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.025018738582730293, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0138345956802368, "metrics/particle/momentum/class_3/accepted/samples": 10858.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4096981883049011, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.966568112373352, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0276904106140137, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.884041428565979, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21028195321559906, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.3206729888916, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01217953022569418, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7571871876716614, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.017372597008943558, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.704604387283325, "metrics/particle/momentum/class_3/assigned/samples": 14861.0, "metrics/particle/momentum/class_3/efficiency": 0.7045159339904785, "metrics/particle/momentum/class_3/event/energy_bias": 10.408112525939941, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.39046859741211, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.518290996551514, "metrics/particle/momentum/class_3/event/px_bias": -0.24005553126335144, "metrics/particle/momentum/class_3/event/py_bias": 0.7130773067474365, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.285774230957031, "metrics/particle/momentum/class_3/event/pz_bias": -0.6215937733650208, "metrics/particle/momentum/class_3/event/pz_residual_rms": 17.465431213378906, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01697484590113163, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11248909682035446, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05854591354727745, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.014492414891719818, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10036744922399521, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0029761905316263437, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.41280582547187805, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.011551559902727604, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.363057255744934, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11607076227664948, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5666706562042236, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.159767985343933, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1141610145568848, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.619278907775879, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005524862091988325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.5050672292709351, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0947011262178421, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6074939966201782, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.9341772794723511, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.165825128555298, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4114930629730225, "metrics/particle/momentum/class_4/event/px_bias": -0.025712424889206886, "metrics/particle/momentum/class_4/event/py_bias": 0.07953961193561554, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.376867413520813, "metrics/particle/momentum/class_4/event/pz_bias": 0.28482508659362793, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.521533489227295, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013591398485004902, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.030002133920788765, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02360880933701992, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004711108282208443, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06322062015533447, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3131949305534363, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.17421399056911469, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7893232703208923, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.034149881452322006, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03817768022418022, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.031719058752059937, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0037103474605828524, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10392657667398453, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6491308212280273, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.511677086353302, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.14509391784668, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -1.443331003189087, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7797191143035889, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2410635948181152, "metrics/particle/momentum/class_5/event/px_bias": 0.12704616785049438, "metrics/particle/momentum/class_5/event/py_bias": 0.07347807288169861, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0373454093933105, "metrics/particle/momentum/class_5/event/pz_bias": -0.4704764485359192, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.573177337646484, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3320417404174805, "step_valid_time": 2024.9716250896454, "step_total_time": 2026.3036675453186, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.01445759065942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09303371339549071, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2835, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2822, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9402985074626866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402985074626866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9402985074626866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05970149253731343, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9359867330016584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9359867330016584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9359867330016584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9402985074626866, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0183725238549228, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11410065880763454, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2230, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2080, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9280958721704394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9376681614349776, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9328574615213027, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06233183856502242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9232134931202841, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9327354260089686, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9279500334597368, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9280958721704394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.024486957510155, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11427737193556964, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1153, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8972762645914397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9043137254901961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.90078125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09568627450980392, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.890272373540856, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8972549019607843, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.89375, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8972762645914397} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_11000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..3d07016d9038d5244a3c0d5a385ab74cc96fdd3d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5211294296001516, "train": {"Classification_binary": 3.0057790279388428, "Classification": 0.025214534252882004, "Regression_pt": 0.16105610132217407, "Regression_eta": 0.010558263398706913, "Regression_sin_phi": 0.009314804337918758, "Regression_cos_phi": 0.010481745935976505, "Regression_energy": 0.1619705855846405, "Total": 10.14780044555664, "Total_unweighted": 3.3843750627711415}, "train_group": {}, "valid": {"Classification_binary": 3.0343077977498374, "Classification": 0.024925092856089275, "Regression_pt": 0.1587935189406077, "Regression_eta": 0.011893046398957571, "Regression_sin_phi": 0.009624634558955828, "Regression_cos_phi": 0.009491554771860441, "Regression_energy": 0.16132898132006326, "Total": 10.151381174723307, "Total_unweighted": 3.4103646265963716, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.025560922920703888, "diagnostic/loss/clic_pf/Classification_binary": 3.042940139770508, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009257132187485695, "diagnostic/loss/clic_pf/Regression_energy": 0.16517266631126404, "diagnostic/loss/clic_pf/Regression_eta": 0.011049605906009674, "diagnostic/loss/clic_pf/Regression_pt": 0.16258791089057922, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009343164041638374, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05235201492905617, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2080526053905487, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06790563464164734, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4694318175315857, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.008650224655866623, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20694515109062195, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07749476283788681, "diagnostic/regression/clic_pf/pt_residual_rms": 0.47371581196784973, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9581171870231628, "metrics/particle/class_1/pid_accuracy": 0.9855601787567139, "metrics/particle/class_2/efficiency": 0.37776273488998413, "metrics/particle/class_2/pid_accuracy": 0.68597012758255, "metrics/particle/class_3/efficiency": 0.719603419303894, "metrics/particle/class_3/pid_accuracy": 0.9811597466468811, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.2604602575302124, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.6209016442298889, "metrics/particle/count/bias_mean": 0.30588236451148987, "metrics/particle/count/mae": 3.535947799682617, "metrics/particle/count/prediction_mean": 70.30522918701172, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06532090157270432, "metrics/particle/event/energy_response_mean": 1.0249031782150269, "metrics/particle/event/met_abs_error": 6.497816562652588, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0534040667116642, "metrics/particle/event/scalar_pt_response_mean": 1.0261757373809814, "metrics/particle/event/vector_pt_closure": 0.05191656947135925, "metrics/particle/matched/delta_eta_abs_mean": 0.012489460408687592, "metrics/particle/matched/delta_phi_abs_mean": 0.012330865487456322, "metrics/particle/matched/delta_r_mean": 0.01965820975601673, "metrics/particle/matched/energy_relative_abs_mean": 0.08466049283742905, "metrics/particle/matched/pid_accuracy": 0.9621555805206299, "metrics/particle/matched/pt_relative_abs_mean": 0.07943161576986313, "metrics/particle/matching/duplicate_fraction": 0.03663763031363487, "metrics/particle/matching/efficiency": 0.8065434694290161, "metrics/particle/matching/f1": 0.8047851324081421, "metrics/particle/matching/purity": 0.8030343651771545, "metrics/particle/matching/target_coverage_dr0p05": 0.7709782719612122, "metrics/particle/matching/target_coverage_dr0p10": 0.8446484208106995, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7120141386985779, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8486914038658142, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023298555985093117, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08502914756536484, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0810290202498436, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019084332510828972, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1367732137441635, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0033836858347058296, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3593425452709198, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03732803463935852, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.227138638496399, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8275.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.34461042284965515, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.004117727279663, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.192129373550415, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0667366981506348, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1517874151468277, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.708267211914062, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016153698787093163, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8796804547309875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.03267098218202591, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.326132297515869, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10462.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7698390483856201, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.3950138092041016, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.298324584960938, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.620789051055908, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.15020152926445007, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.038672879338264465, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.179860591888428, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.9388657212257385, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.640189170837402, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.031038491055369377, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0988692194223404, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09604804962873459, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014163913205265999, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1543535590171814, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021851638332009315, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9964466094970703, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05801010876893997, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2567739486694336, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1739.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7940208911895752, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.220249891281128, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 61080.67578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 61080.4921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2791983485221863, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3285653.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08396682888269424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.354914426803589, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07984241098165512, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.00978946685791, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2894.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5660807490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.43196234107017517, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.765363693237305, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.46625816822052, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.06907345354557037, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.17279218137264252, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.554950714111328, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.39937758445739746, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.011571884155273, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017054514959454536, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08393144607543945, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07763317227363586, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01551373302936554, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13335810601711273, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009042553137987852, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1261022686958313, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0002373626339249313, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6367493867874146, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18800.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1742279827594757, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8652742505073547, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6905279755592346, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5815837979316711, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10289151221513748, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.073199272155762, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002822114387527108, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.31051769852638245, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0013554611941799521, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.2298061847686768, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21615.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8563359975814819, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.318000316619873, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.20840835571289, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.595396995544434, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5197312831878662, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.08454775810241699, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.541203498840332, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.19981180131435394, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 20.062284469604492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019691690802574158, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08514821529388428, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07971980422735214, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01645769737660885, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13569779694080353, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028805441688746214, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24561333656311035, "metrics/particle/momentum/all/accepted/pz_bias": 0.014066311530768871, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1563477516174316, "metrics/particle/momentum/all/accepted/samples": 28814.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2764904797077179, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0189409255981445, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5055.470703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5055.35107421875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13210946321487427, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 945185.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013525492511689663, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.732726514339447, "metrics/particle/momentum/all/assigned/pz_bias": -0.004004408605396748, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.448416233062744, "metrics/particle/momentum/all/assigned/samples": 34971.0, "metrics/particle/momentum/all/efficiency": 0.8054227828979492, "metrics/particle/momentum/all/event/energy_bias": 7.281052589416504, "metrics/particle/momentum/all/event/p3_error_mean": 19.04751205444336, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.531020164489746, "metrics/particle/momentum/all/event/px_bias": -0.30045631527900696, "metrics/particle/momentum/all/event/py_bias": -0.04957154393196106, "metrics/particle/momentum/all/event/pz_abs_mean": 14.314522743225098, "metrics/particle/momentum/all/event/pz_bias": -1.1384315490722656, "metrics/particle/momentum/all/event/pz_residual_rms": 29.281879425048828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011950650252401829, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.036023881286382675, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.027234381064772606, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004283226560801268, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06766090542078018, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012400794075801969, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13016796112060547, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.027794789522886276, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8020626902580261, "metrics/particle/momentum/class_1/accepted/samples": 16128.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06465784460306168, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.17592425644397736, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.16392028331756592, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12709888815879822, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.028151484206318855, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.033681631088257, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005274699069559574, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31816422939300537, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02787320502102375, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.071789741516113, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9558466076850891, "metrics/particle/momentum/class_1/event/energy_bias": 6.79164981842041, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.951282501220703, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.778561115264893, "metrics/particle/momentum/class_1/event/px_bias": -1.6210391521453857, "metrics/particle/momentum/class_1/event/py_bias": -1.1409589052200317, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.645476341247559, "metrics/particle/momentum/class_1/event/pz_bias": 0.7507801651954651, "metrics/particle/momentum/class_1/event/pz_residual_rms": 25.977020263671875, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04085375368595123, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2048339992761612, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20428746938705444, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.033647336065769196, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24575595557689667, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.038188278675079346, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.853486180305481, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.07570726424455643, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.988574743270874, "metrics/particle/momentum/class_2/accepted/samples": 1126.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0017749071121216, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8603548407554626, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 65954.3203125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 65953.78125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4201492667198181, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3414315.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08208955079317093, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4662926197052, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06650694459676743, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.976099967956543, "metrics/particle/momentum/class_2/assigned/samples": 2680.0, "metrics/particle/momentum/class_2/efficiency": 0.38221317529678345, "metrics/particle/momentum/class_2/event/energy_bias": -5.377592086791992, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.859886169433594, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.87452507019043, "metrics/particle/momentum/class_2/event/px_bias": 0.6503701210021973, "metrics/particle/momentum/class_2/event/py_bias": 0.19859623908996582, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.235013961791992, "metrics/particle/momentum/class_2/event/pz_bias": -0.6659914255142212, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.32775115966797, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02910785749554634, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14451490342617035, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14509978890419006, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.039753034710884094, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18644827604293823, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018104462651535869, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24667581915855408, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0007149679586291313, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9382944703102112, "metrics/particle/momentum/class_3/accepted/samples": 11047.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.39393529295921326, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9723409414291382, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2680721282958984, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.126431703567505, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2014925330877304, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.335739135742188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010824257507920265, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7178252339363098, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.025472665205597878, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.997257947921753, "metrics/particle/momentum/class_3/assigned/samples": 14874.0, "metrics/particle/momentum/class_3/efficiency": 0.7167791128158569, "metrics/particle/momentum/class_3/event/energy_bias": 7.207525253295898, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.518973350524902, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.490717887878418, "metrics/particle/momentum/class_3/event/px_bias": 0.4136037230491638, "metrics/particle/momentum/class_3/event/py_bias": 0.5105383992195129, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.327048301696777, "metrics/particle/momentum/class_3/event/pz_bias": -1.4016387462615967, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.294766426086426, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014864210970699787, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12122529745101929, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.061543017625808716, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.012689951807260513, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10698283463716507, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.36478129029273987, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.13627660274505615, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.127977967262268, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08155164122581482, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.80220627784729, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.210508346557617, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1586618423461914, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.632631301879883, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4947783052921295, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.018189050257205963, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.6181358098983765, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -1.0398533344268799, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.184021472930908, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3863168954849243, "metrics/particle/momentum/class_4/event/px_bias": 0.10080258548259735, "metrics/particle/momentum/class_4/event/py_bias": 0.0985024943947792, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.430474042892456, "metrics/particle/momentum/class_4/event/pz_bias": 0.4107213020324707, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.842978477478027, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011915488168597221, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026729976758360863, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023866891860961914, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.002156629227101803, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0702076181769371, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.24437709152698517, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.025417182594537735, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.702894389629364, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.024748707190155983, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04542398080229759, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04271870478987694, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006570945959538221, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14878779649734497, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6048778891563416, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.32861635088920593, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.175110816955566, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.30067750811576843, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5568370819091797, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1231480836868286, "metrics/particle/momentum/class_5/event/px_bias": 0.15580648183822632, "metrics/particle/momentum/class_5/event/py_bias": 0.2837502360343933, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8489184379577637, "metrics/particle/momentum/class_5/event/pz_bias": -0.23230277001857758, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7908883094787598, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2606546878814697, "step_valid_time": 2206.8057777881622, "step_total_time": 2208.0664327144623, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005137685750635, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09379996839181826, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3011, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2848, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9446102819237148, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9458651610760545, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9452373050116163, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054134838923945536, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9412935323383085, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9425440053138492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9419183538001992, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9446102819237148, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0129502342432741, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10607791797789512, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2098, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9325343985796716, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9358574610244988, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9341929746554024, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06414253897550111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.931202840656902, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9345211581291759, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9328590484659849, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9325343985796716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0123932881029287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10230515359944836, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1268, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1159, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9081712062256809, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9203470031545742, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9142185663924794, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07965299684542587, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9019455252918288, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9140378548895899, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9079514296905601, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9081712062256809} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_12000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..7baa13f7d7bb80b31250c3ccbb5bed6457e053e5 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5685048322910745, "train": {"Classification_binary": 2.897038698196411, "Classification": 0.023794706910848618, "Regression_pt": 0.14744573831558228, "Regression_eta": 0.009677211754024029, "Regression_sin_phi": 0.0074959807097911835, "Regression_cos_phi": 0.008455151692032814, "Regression_energy": 0.1491406112909317, "Total": 9.327951431274414, "Total_unweighted": 3.2430480988696218}, "train_group": {}, "valid": {"Classification_binary": 2.9833412170410156, "Classification": 0.024438798427581787, "Regression_pt": 0.15460596481959024, "Regression_eta": 0.0121468057235082, "Regression_sin_phi": 0.009512888888518015, "Regression_cos_phi": 0.009349410732587179, "Regression_energy": 0.15773979822794595, "Total": 9.96839968363444, "Total_unweighted": 3.351134883860747, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.025401359423995018, "diagnostic/loss/clic_pf/Classification_binary": 3.0005993843078613, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008975747972726822, "diagnostic/loss/clic_pf/Regression_energy": 0.16190937161445618, "diagnostic/loss/clic_pf/Regression_eta": 0.011354758404195309, "diagnostic/loss/clic_pf/Regression_pt": 0.15879635512828827, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009250825271010399, "diagnostic/regression/clic_pf/energy_pred_mean": 0.030737649649381638, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2039225846529007, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04629127308726311, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4605630338191986, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.006165285594761372, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20209012925624847, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0626792460680008, "diagnostic/regression/clic_pf/pt_residual_rms": 0.46556639671325684, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9602838754653931, "metrics/particle/class_1/pid_accuracy": 0.9853029251098633, "metrics/particle/class_2/efficiency": 0.3849796950817108, "metrics/particle/class_2/pid_accuracy": 0.6804335117340088, "metrics/particle/class_3/efficiency": 0.7276214957237244, "metrics/particle/class_3/pid_accuracy": 0.9813674092292786, "metrics/particle/class_4/efficiency": 0.9312015771865845, "metrics/particle/class_4/pid_accuracy": 0.3080124855041504, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.6290983557701111, "metrics/particle/count/bias_mean": 1.0385620594024658, "metrics/particle/count/mae": 3.706536054611206, "metrics/particle/count/prediction_mean": 71.03791046142578, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05701804533600807, "metrics/particle/event/energy_response_mean": 0.9926519393920898, "metrics/particle/event/met_abs_error": 5.43778133392334, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04625805467367172, "metrics/particle/event/scalar_pt_response_mean": 1.003279685974121, "metrics/particle/event/vector_pt_closure": 0.048422086983919144, "metrics/particle/matched/delta_eta_abs_mean": 0.01479269191622734, "metrics/particle/matched/delta_phi_abs_mean": 0.011930101551115513, "metrics/particle/matched/delta_r_mean": 0.021337447687983513, "metrics/particle/matched/energy_relative_abs_mean": 0.08484082669019699, "metrics/particle/matched/pid_accuracy": 0.9623144865036011, "metrics/particle/matched/pt_relative_abs_mean": 0.07856111228466034, "metrics/particle/matching/duplicate_fraction": 0.032607097178697586, "metrics/particle/matching/efficiency": 0.8116788864135742, "metrics/particle/matching/f1": 0.8057019114494324, "metrics/particle/matching/purity": 0.7998123168945312, "metrics/particle/matching/target_coverage_dr0p05": 0.7453290820121765, "metrics/particle/matching/target_coverage_dr0p10": 0.8446017503738403, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7179431915283203, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8573749661445618, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027081551030278206, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08488786965608597, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07887986302375793, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005598840769380331, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13631707429885864, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0036088055931031704, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3681553304195404, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.03199034556746483, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2972873449325562, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8313.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32480230927467346, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9021399617195129, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0084189176559448, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8662353157997131, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14431416988372803, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.966265678405762, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016490325331687927, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8237908482551575, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.040376342833042145, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6693432331085205, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10491.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7733742594718933, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3074794411659241, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.048297882080078, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.579880714416504, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.04728987067937851, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1421625167131424, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.9182281494140625, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.131460428237915, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.24904727935791, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.034640420228242874, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09622492641210556, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09220251441001892, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0036462543066591024, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15059076249599457, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016187049448490143, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9654721021652222, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.13126571476459503, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.582862377166748, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1668.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8790149092674255, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3810268640518188, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10940.701171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10940.498046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2967988848686218, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 586381.6875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07794015109539032, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1934139728546143, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13699327409267426, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.25163459777832, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2874.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.54296875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.6292147636413574, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.787663459777832, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4077826738357544, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1128326803445816, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.13795176148414612, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.606913566589355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5856322050094604, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.088537216186523, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017690327018499374, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08399605005979538, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07727093994617462, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006301006302237511, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13369053602218628, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008428593864664435, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.123384989798069, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.014676692895591259, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6080301403999329, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18983.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1751573234796524, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7558588981628418, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5519294142723083, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42916107177734375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09604206681251526, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.878583908081055, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029061720706522465, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.28211477398872375, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.020668918266892433, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.727610468864441, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21678.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8646715879440308, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5173847675323486, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.700082778930664, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.322560787200928, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6017316579818726, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7891488671302795, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8002190589904785, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6858053803443909, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.252998352050781, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021361852064728737, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08495625108480453, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07859261333942413, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005946592427790165, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13547466695308685, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002520370064303279, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24213163554668427, "metrics/particle/momentum/all/accepted/pz_bias": -0.007996932603418827, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2102024555206299, "metrics/particle/momentum/all/accepted/samples": 28964.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2776831090450287, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8509240746498108, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 897.9287719726562, "metrics/particle/momentum/all/assigned/pt_relative_bias": 897.7935791015625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12695831060409546, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 167928.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013126729987561703, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6830450296401978, "metrics/particle/momentum/all/assigned/pz_bias": -0.011933659203350544, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3165132999420166, "metrics/particle/momentum/all/assigned/samples": 35043.0, "metrics/particle/momentum/all/efficiency": 0.8096156716346741, "metrics/particle/momentum/all/event/energy_bias": -3.419309616088867, "metrics/particle/momentum/all/event/p3_error_mean": 16.867727279663086, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.077779769897461, "metrics/particle/momentum/all/event/px_bias": -0.7618541717529297, "metrics/particle/momentum/all/event/py_bias": -0.5090346336364746, "metrics/particle/momentum/all/event/pz_abs_mean": 12.857501029968262, "metrics/particle/momentum/all/event/pz_bias": -2.402897834777832, "metrics/particle/momentum/all/event/pz_residual_rms": 22.46005630493164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011413196101784706, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03607824817299843, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.024619171395897865, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.00268527097068727, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06430129706859589, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001236552488990128, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1444261074066162, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.010752866044640541, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8664587140083313, "metrics/particle/momentum/class_1/accepted/samples": 16174.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06431453675031662, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16520307958126068, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15702185034751892, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11539965122938156, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02732175588607788, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.927449941635132, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005156166851520538, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.30511394143104553, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.003451754106208682, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.618007183074951, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9585728645324707, "metrics/particle/momentum/class_1/event/energy_bias": 1.0890601873397827, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.29904556274414, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.817267417907715, "metrics/particle/momentum/class_1/event/px_bias": -0.9726368188858032, "metrics/particle/momentum/class_1/event/py_bias": -0.3092917203903198, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.421541213989258, "metrics/particle/momentum/class_1/event/pz_bias": -0.6985886693000793, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.164447784423828, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04158293083310127, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20455238223075867, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20557890832424164, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04801630601286888, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24571262300014496, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03205699101090431, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.628231167793274, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.005699660629034042, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4708335399627686, "metrics/particle/momentum/class_2/accepted/samples": 1123.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0332427024841309, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7428396940231323, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11656.6484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11656.0751953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.41082683205604553, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 605317.5625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08713385462760925, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4953203201293945, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07514514774084091, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.190312385559082, "metrics/particle/momentum/class_2/assigned/samples": 2697.0, "metrics/particle/momentum/class_2/efficiency": 0.3811948299407959, "metrics/particle/momentum/class_2/event/energy_bias": -10.344647407531738, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.246173858642578, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.123642921447754, "metrics/particle/momentum/class_2/event/px_bias": -0.16438685357570648, "metrics/particle/momentum/class_2/event/py_bias": -0.8452829122543335, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.268657684326172, "metrics/particle/momentum/class_2/event/pz_bias": -1.1484804153442383, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.010032653808594, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.034117892384529114, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14425653219223022, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14585596323013306, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024367505684494972, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1877049058675766, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0015239802887663245, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24363039433956146, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.003008500672876835, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.211678147315979, "metrics/particle/momentum/class_3/accepted/samples": 11155.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.39013317227363586, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6578607559204102, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6981221437454224, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5335816144943237, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19143947958946228, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.68787956237793, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009176770225167274, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6109477877616882, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.011560029350221157, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3968496322631836, "metrics/particle/momentum/class_3/assigned/samples": 14929.0, "metrics/particle/momentum/class_3/efficiency": 0.7237866520881653, "metrics/particle/momentum/class_3/event/energy_bias": 6.774174690246582, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.899293899536133, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.009425163269043, "metrics/particle/momentum/class_3/event/px_bias": 0.062341004610061646, "metrics/particle/momentum/class_3/event/py_bias": 0.4483140707015991, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.083978652954102, "metrics/particle/momentum/class_3/event/pz_bias": -0.7110923528671265, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.917167663574219, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015019956044852734, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10057670623064041, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04879939556121826, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.004353262018412352, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09603375941514969, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23777690529823303, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.031083283945918083, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8288949131965637, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09055379778146744, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.7442746758460999, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.350513219833374, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.2871832847595215, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.4570231437683105, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.35679879784584045, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04598904773592949, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1640437841415405, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.5489628314971924, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.224360466003418, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.44682776927948, "metrics/particle/momentum/class_4/event/px_bias": 0.16693635284900665, "metrics/particle/momentum/class_4/event/py_bias": 0.10999185591936111, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4148293733596802, "metrics/particle/momentum/class_4/event/pz_bias": 0.3740032911300659, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.789425373077393, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010270853526890278, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026162002235651016, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022350966930389404, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.004351412411779165, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06732524931430817, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.28951746225357056, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.15802006423473358, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8946310877799988, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01064081210643053, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04608813300728798, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04223420098423958, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0009600788471288979, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15609896183013916, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.609225869178772, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.46449705958366394, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7985968589782715, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3889334797859192, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5173921585083008, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.071158766746521, "metrics/particle/momentum/class_5/event/px_bias": 0.14589214324951172, "metrics/particle/momentum/class_5/event/py_bias": 0.08723410964012146, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8593228459358215, "metrics/particle/momentum/class_5/event/pz_bias": -0.21873971819877625, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.8408570289611816, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4384229183197021, "step_valid_time": 2384.987799167633, "step_total_time": 2386.4262223243713, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9975520259806625, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07526046559258237, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3016, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2858, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2853, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.947927031509121, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9476127320954907, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9477698557453159, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05238726790450928, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9462686567164179, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9459549071618037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9461117559277068, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.947927031509121, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962789083982754, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09255683608706489, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2099, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2086, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9316466932978251, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9345503116651825, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.933096243609691, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06544968833481746, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9258766089658234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9287622439893143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9273171815959103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9316466932978251, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9984111171905409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09357234622723265, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1161, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9035019455252918, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9077404222048475, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.905616224648986, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09225957779515247, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8996108949416343, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9038311180609852, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9017160686427457, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9035019455252918} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_13000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..90616ed293199eb1453c625ef20abe6df5b2f5d3 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6158802349819974, "train": {"Classification_binary": 3.042537212371826, "Classification": 0.02473532222211361, "Regression_pt": 0.16732941567897797, "Regression_eta": 0.011096234433352947, "Regression_sin_phi": 0.010681587271392345, "Regression_cos_phi": 0.010367578826844692, "Regression_energy": 0.16806483268737793, "Total": 10.466602325439453, "Total_unweighted": 3.4348121834918857}, "train_group": {}, "valid": {"Classification_binary": 2.923853874206543, "Classification": 0.024303195377190907, "Regression_pt": 0.1540290911992391, "Regression_eta": 0.011962026357650757, "Regression_sin_phi": 0.00935569778084755, "Regression_cos_phi": 0.009398857752482096, "Regression_energy": 0.15671005845069885, "Total": 9.864512125651041, "Total_unweighted": 3.2896128011246524, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02499484270811081, "diagnostic/loss/clic_pf/Classification_binary": 2.9259519577026367, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00912462454289198, "diagnostic/loss/clic_pf/Regression_energy": 0.16051246225833893, "diagnostic/loss/clic_pf/Regression_eta": 0.011215368285775185, "diagnostic/loss/clic_pf/Regression_pt": 0.1575881540775299, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009255344979465008, "diagnostic/regression/clic_pf/energy_pred_mean": 0.049200233072042465, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20637530088424683, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06475385278463364, "diagnostic/regression/clic_pf/energy_residual_rms": 0.46528440713882446, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.01071862317621708, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20470523834228516, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07956316322088242, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4721693992614746, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9595881104469299, "metrics/particle/class_1/pid_accuracy": 0.986410915851593, "metrics/particle/class_2/efficiency": 0.3792286813259125, "metrics/particle/class_2/pid_accuracy": 0.6975914239883423, "metrics/particle/class_3/efficiency": 0.7153357267379761, "metrics/particle/class_3/pid_accuracy": 0.9801735281944275, "metrics/particle/class_4/efficiency": 0.9215116500854492, "metrics/particle/class_4/pid_accuracy": 0.29442691802978516, "metrics/particle/class_5/efficiency": 0.9798792600631714, "metrics/particle/class_5/pid_accuracy": 0.5954825282096863, "metrics/particle/count/bias_mean": -0.8111110925674438, "metrics/particle/count/mae": 3.5771241188049316, "metrics/particle/count/prediction_mean": 69.188232421875, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05951225012540817, "metrics/particle/event/energy_response_mean": 0.9897089004516602, "metrics/particle/event/met_abs_error": 5.539322853088379, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04673877730965614, "metrics/particle/event/scalar_pt_response_mean": 1.0027821063995361, "metrics/particle/event/vector_pt_closure": 0.0496043935418129, "metrics/particle/matched/delta_eta_abs_mean": 0.011906111612915993, "metrics/particle/matched/delta_phi_abs_mean": 0.01159753929823637, "metrics/particle/matched/delta_r_mean": 0.01861731894314289, "metrics/particle/matched/energy_relative_abs_mean": 0.08364172279834747, "metrics/particle/matched/pid_accuracy": 0.9629157185554504, "metrics/particle/matched/pt_relative_abs_mean": 0.07729966193437576, "metrics/particle/matching/duplicate_fraction": 0.0326002761721611, "metrics/particle/matching/efficiency": 0.8054510354995728, "metrics/particle/matching/f1": 0.8101447820663452, "metrics/particle/matching/purity": 0.8148935437202454, "metrics/particle/matching/target_coverage_dr0p05": 0.7592787742614746, "metrics/particle/matching/target_coverage_dr0p10": 0.8395223021507263, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7137134671211243, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8475802540779114, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0216992050409317, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08293928951025009, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0784340426325798, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0131485341116786, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13581199944019318, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004006313160061836, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37234073877334595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0044085499830543995, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3797407150268555, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8237.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33188655972480774, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8904910683631897, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9286516308784485, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8030633926391602, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.141097754240036, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.398417472839355, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016097750514745712, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8550346493721008, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.038994308561086655, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.7942700386047363, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10312.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7663038372993469, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.2757236957550049, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.880227088928223, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5354883670806885, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5852400660514832, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.03967245668172836, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.756233215332031, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.4763529300689697, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.596439361572266, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028958551585674286, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09592632949352264, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09476315230131149, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011514483951032162, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15549728274345398, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02144133485853672, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0437406301498413, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.02529921382665634, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.383046865463257, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1679.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8267735242843628, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7309746742248535, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 58628.25390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 58628.0859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.28776460886001587, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3094906.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0810907781124115, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.146200180053711, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.10326208919286728, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.510747909545898, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2787.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5465494990348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.117520809173584, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.768458843231201, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2780954837799072, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07892434298992157, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.03775807097554207, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.595184326171875, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.8156571388244629, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.440156936645508, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01649228110909462, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0844421312212944, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07644254714250565, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012152629904448986, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.133417546749115, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006880491273477674, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11716006696224213, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0036987962666898966, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5829023718833923, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18894.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15991955995559692, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7351759076118469, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5569022297859192, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.44679272174835205, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09479597210884094, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.057438850402832, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028782321605831385, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2665002942085266, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0266125351190567, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3202778100967407, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21541.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8606176376342773, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8717483282089233, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.9574294090271, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.63291072845459, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.35109978914260864, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.13561692833900452, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.782986640930176, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.12825444340705872, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.707794189453125, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01870749518275261, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08468174189329147, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07807961851358414, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012400176376104355, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13548550009727478, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028462340123951435, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24411779642105103, "metrics/particle/momentum/all/accepted/pz_bias": -0.005160557571798563, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1975406408309937, "metrics/particle/momentum/all/accepted/samples": 28810.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2647649943828583, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8615298867225647, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4717.62451171875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4717.505859375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12410508096218109, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 877863.6875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013106235302984715, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.673390805721283, "metrics/particle/momentum/all/assigned/pz_bias": -0.013248924165964127, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.556300640106201, "metrics/particle/momentum/all/assigned/samples": 34640.0, "metrics/particle/momentum/all/efficiency": 0.8053109645843506, "metrics/particle/momentum/all/event/energy_bias": -4.521496295928955, "metrics/particle/momentum/all/event/p3_error_mean": 16.8955078125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.394376754760742, "metrics/particle/momentum/all/event/px_bias": -1.0152641534805298, "metrics/particle/momentum/all/event/py_bias": 0.21304744482040405, "metrics/particle/momentum/all/event/pz_abs_mean": 12.684240341186523, "metrics/particle/momentum/all/event/pz_bias": -2.420264482498169, "metrics/particle/momentum/all/event/pz_residual_rms": 21.98360252380371, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009097419679164886, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.036590877920389175, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.025467634201049805, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006237291265279055, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06868261843919754, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014226511120796204, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12950880825519562, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.009628511033952236, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8379471898078918, "metrics/particle/momentum/class_1/accepted/samples": 16167.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0556948147714138, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1642732322216034, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14927467703819275, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11682053655385971, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.027680635452270508, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.580674886703491, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004682591650635004, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.266696035861969, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.020545221865177155, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1649744510650635, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.958158016204834, "metrics/particle/momentum/class_1/event/energy_bias": 4.554227352142334, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.24107551574707, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.790925979614258, "metrics/particle/momentum/class_1/event/px_bias": -0.7453104257583618, "metrics/particle/momentum/class_1/event/py_bias": -0.7173952460289001, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.4473876953125, "metrics/particle/momentum/class_1/event/pz_bias": 0.3991175591945648, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.771451950073242, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04252117872238159, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2052386999130249, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20644530653953552, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04996492713689804, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24740922451019287, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.036509349942207336, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8100388050079346, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06126299127936363, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.049065589904785, "metrics/particle/momentum/class_2/accepted/samples": 1123.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.017697811126709, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8381535410881042, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 63427.43359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 63426.90625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3928571343421936, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3219164.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08423913270235062, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.461012601852417, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08901681005954742, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.588316917419434, "metrics/particle/momentum/class_2/assigned/samples": 2576.0, "metrics/particle/momentum/class_2/efficiency": 0.3811948299407959, "metrics/particle/momentum/class_2/event/energy_bias": -10.280452728271484, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.757159233093262, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.004417419433594, "metrics/particle/momentum/class_2/event/px_bias": 0.1413920521736145, "metrics/particle/momentum/class_2/event/py_bias": 0.10644347965717316, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.740966796875, "metrics/particle/momentum/class_2/event/pz_bias": -0.8536299467086792, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.473426818847656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0307333804666996, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14286261796951294, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14372624456882477, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.028028013184666634, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1856042742729187, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016351743834093213, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.252023845911026, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.021124692633748055, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0049474239349365, "metrics/particle/momentum/class_3/accepted/samples": 11008.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.38041651248931885, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6280089616775513, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.780987024307251, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.630581259727478, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1906614750623703, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.809619903564453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010717455297708511, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6601510047912598, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06579752266407013, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.359283685684204, "metrics/particle/momentum/class_3/assigned/samples": 14649.0, "metrics/particle/momentum/class_3/efficiency": 0.7142486572265625, "metrics/particle/momentum/class_3/event/energy_bias": 2.50211501121521, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.202674865722656, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.690499782562256, "metrics/particle/momentum/class_3/event/px_bias": -0.5499090552330017, "metrics/particle/momentum/class_3/event/py_bias": 0.6950594782829285, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.433897018432617, "metrics/particle/momentum/class_3/event/pz_bias": -2.21412992477417, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.599390029907227, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013287573121488094, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12013315409421921, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05858350917696953, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.012651593424379826, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1033124029636383, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.25815171003341675, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.022745288908481598, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8603202104568481, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08945606648921967, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.9171242713928223, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.315915822982788, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.267852544784546, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0635359138250351, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 34.52300262451172, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3601672351360321, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.004187564831227064, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.160443902015686, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.5827164053916931, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0718493461608887, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3373217582702637, "metrics/particle/momentum/class_4/event/px_bias": 0.03358849510550499, "metrics/particle/momentum/class_4/event/py_bias": -0.05647795647382736, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3372232913970947, "metrics/particle/momentum/class_4/event/pz_bias": 0.23844240605831146, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.627408504486084, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007767276838421822, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02739839069545269, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023569485172629356, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0013293384108692408, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07723542302846909, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.25893187522888184, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05953807011246681, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.0300610065460205, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02822680026292801, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.044802192598581314, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04112713783979416, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006131066009402275, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14727577567100525, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6062085628509521, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.39844223856925964, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.208359241485596, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.7146694660186768, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.4619120359420776, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9234614372253418, "metrics/particle/momentum/class_5/event/px_bias": 0.10497479140758514, "metrics/particle/momentum/class_5/event/py_bias": 0.18541763722896576, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9531145095825195, "metrics/particle/momentum/class_5/event/pz_bias": 0.009935498237609863, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.139164447784424, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3652276992797852, "step_valid_time": 2560.1747217178345, "step_total_time": 2561.539949655533, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968979339704831, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07255326307908849, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3013, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2846, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9439469320066335, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9445735147693329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9442601194426012, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05542648523066711, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9406301824212272, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9412545635579157, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9409422694094227, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9439469320066335, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9951164163251881, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09220212099164915, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2084, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2078, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9249889036839769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9328558639212176, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9289057276576778, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06714413607878246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9223257878384377, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9301700984780662, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9262313349676844, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9249889036839769, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9956868711090183, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09629060301492831, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1146, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1143, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8918287937743191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8946135831381733, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8932190179267342, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1053864168618267, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8894941634241245, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.892271662763466, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8908807482462977, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8918287937743191} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_14000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..cfce4ed1412439e5fdb2b7f521ce59ee28961cf7 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6632556376729202, "train": {"Classification_binary": 2.907382011413574, "Classification": 0.02138492651283741, "Regression_pt": 0.15330412983894348, "Regression_eta": 0.013484392315149307, "Regression_sin_phi": 0.009956607595086098, "Regression_cos_phi": 0.010335731320083141, "Regression_energy": 0.16006188094615936, "Total": 9.945804595947266, "Total_unweighted": 3.275909679941833}, "train_group": {}, "valid": {"Classification_binary": 2.922610600789388, "Classification": 0.02391185611486435, "Regression_pt": 0.15634225805600485, "Regression_eta": 0.012289983530839285, "Regression_sin_phi": 0.009433882931868235, "Regression_cos_phi": 0.00923207588493824, "Regression_energy": 0.1599742571512858, "Total": 9.938617706298828, "Total_unweighted": 3.293794914459189, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.024609623476862907, "diagnostic/loss/clic_pf/Classification_binary": 2.927910566329956, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008916014805436134, "diagnostic/loss/clic_pf/Regression_energy": 0.16313493251800537, "diagnostic/loss/clic_pf/Regression_eta": 0.011408270336687565, "diagnostic/loss/clic_pf/Regression_pt": 0.15921500325202942, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009345336817204952, "diagnostic/regression/clic_pf/energy_pred_mean": 0.042599182575941086, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20812688767910004, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05815280228853226, "diagnostic/regression/clic_pf/energy_residual_rms": 0.45528438687324524, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.001121620531193912, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2052057683467865, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06772290915250778, "diagnostic/regression/clic_pf/pt_residual_rms": 0.45887693762779236, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9525513052940369, "metrics/particle/class_1/pid_accuracy": 0.9864148497581482, "metrics/particle/class_2/efficiency": 0.3762967884540558, "metrics/particle/class_2/pid_accuracy": 0.6640695333480835, "metrics/particle/class_3/efficiency": 0.7260480523109436, "metrics/particle/class_3/pid_accuracy": 0.9835238456726074, "metrics/particle/class_4/efficiency": 0.9001938104629517, "metrics/particle/class_4/pid_accuracy": 0.3110871911048889, "metrics/particle/class_5/efficiency": 0.9758551120758057, "metrics/particle/class_5/pid_accuracy": 0.6144329905509949, "metrics/particle/count/bias_mean": -1.0313725471496582, "metrics/particle/count/mae": 3.5490195751190186, "metrics/particle/count/prediction_mean": 68.96797180175781, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0569564625620842, "metrics/particle/event/energy_response_mean": 0.9907188415527344, "metrics/particle/event/met_abs_error": 5.427447319030762, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04565128684043884, "metrics/particle/event/scalar_pt_response_mean": 0.9942190647125244, "metrics/particle/event/vector_pt_closure": 0.047513071447610855, "metrics/particle/matched/delta_eta_abs_mean": 0.015467368066310883, "metrics/particle/matched/delta_phi_abs_mean": 0.013066014274954796, "metrics/particle/matched/delta_r_mean": 0.022478578612208366, "metrics/particle/matched/energy_relative_abs_mean": 0.08963663130998611, "metrics/particle/matched/pid_accuracy": 0.9634767770767212, "metrics/particle/matched/pt_relative_abs_mean": 0.08242057263851166, "metrics/particle/matching/duplicate_fraction": 0.030325716361403465, "metrics/particle/matching/efficiency": 0.8063194155693054, "metrics/particle/matching/f1": 0.8123036623001099, "metrics/particle/matching/purity": 0.8183773756027222, "metrics/particle/matching/target_coverage_dr0p05": 0.7388304471969604, "metrics/particle/matching/target_coverage_dr0p10": 0.840297281742096, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7082045674324036, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8484206199645996, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02446594461798668, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0883006900548935, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0815826803445816, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01423262432217598, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13741154968738556, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003256150521337986, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37682417035102844, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.018867865204811096, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3455982208251953, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8292.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.31302544474601746, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8602204918861389, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9133541584014893, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7817805409431458, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13944917917251587, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.291810989379883, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014546159654855728, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8183591961860657, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02460789680480957, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7172675132751465, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10312.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7714205980300903, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.22493189573287964, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.8413286209106445, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4440841674804688, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.08051832765340805, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.15440523624420166, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.795866012573242, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1115535497665405, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.45368480682373, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03175158053636551, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09585854411125183, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09254232048988342, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012699800543487072, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15148700773715973, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022275738418102264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9761905074119568, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06880928575992584, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2698495388031006, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1661.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7851259112358093, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6393756866455078, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 25858.771484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 25858.595703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2926481068134308, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1368657.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08708065748214722, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2906992435455322, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.026273217052221298, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.552736282348633, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2802.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5406901240348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.4630703926086426, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.51407527923584, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.366098165512085, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.17998886108398438, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08390461653470993, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.318988800048828, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.20780381560325623, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.990772247314453, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020666083320975304, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0896964818239212, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08172351866960526, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01571781001985073, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13676203787326813, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006906444323249161, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1251048743724823, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015768734738230705, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5976119637489319, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18823.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16343563795089722, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6542276740074158, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4909389019012451, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37727516889572144, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09510287642478943, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.42336368560791, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029326877556741238, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.28154587745666504, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.018888603895902634, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.221234083175659, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21482.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8573836088180542, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -1.7901544570922852, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.369316101074219, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.86130952835083, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.1633581817150116, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.13433706760406494, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.1248626708984375, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.21895594894886017, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.656553268432617, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.022400913760066032, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08964996784925461, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08230741322040558, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015115640126168728, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1378410905599594, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002675840863958001, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2467656284570694, "metrics/particle/momentum/all/accepted/pz_bias": -0.01972336135804653, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1715439558029175, "metrics/particle/momentum/all/accepted/samples": 28776.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25837573409080505, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7954168915748596, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2094.93115234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2094.80712890625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12432073056697845, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 389507.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01320961955934763, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6852707266807556, "metrics/particle/momentum/all/assigned/pz_bias": -0.016935601830482483, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.7793829441070557, "metrics/particle/momentum/all/assigned/samples": 34596.0, "metrics/particle/momentum/all/efficiency": 0.804360568523407, "metrics/particle/momentum/all/event/energy_bias": -4.478156566619873, "metrics/particle/momentum/all/event/p3_error_mean": 16.944477081298828, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.377803802490234, "metrics/particle/momentum/all/event/px_bias": -0.4238654375076294, "metrics/particle/momentum/all/event/py_bias": -0.37264689803123474, "metrics/particle/momentum/all/event/pz_abs_mean": 12.752382278442383, "metrics/particle/momentum/all/event/pz_bias": -1.1004014015197754, "metrics/particle/momentum/all/event/pz_residual_rms": 23.70532989501953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013006271794438362, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04211221635341644, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.031256262212991714, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.014218645170331001, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07510101050138474, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013718276750296354, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13945415616035461, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0004438640607986599, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.91740483045578, "metrics/particle/momentum/class_1/accepted/samples": 16037.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06336983293294907, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.15786054730415344, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14894144237041473, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1175999641418457, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03277425467967987, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.8032920360565186, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005333965644240379, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.31431543827056885, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04065458104014397, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.0552875995635986, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9504534006118774, "metrics/particle/momentum/class_1/event/energy_bias": 7.18904972076416, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.819539070129395, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.3852362632751465, "metrics/particle/momentum/class_1/event/px_bias": -0.504916250705719, "metrics/particle/momentum/class_1/event/py_bias": -0.5375432372093201, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.650842666625977, "metrics/particle/momentum/class_1/event/pz_bias": 1.224658727645874, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.70472526550293, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043752897530794144, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20904482901096344, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20969462394714355, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06289321184158325, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24885912239551544, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0358102060854435, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7877445220947266, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.12143131345510483, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.845942497253418, "metrics/particle/momentum/class_2/accepted/samples": 1117.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9866526126861572, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7353827357292175, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 28190.845703125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 28190.28515625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.41206225752830505, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1429098.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08482490479946136, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.3821797370910645, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.120001420378685, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.761641979217529, "metrics/particle/momentum/class_2/assigned/samples": 2570.0, "metrics/particle/momentum/class_2/efficiency": 0.3791581690311432, "metrics/particle/momentum/class_2/event/energy_bias": -14.577524185180664, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.924531936645508, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.010972023010254, "metrics/particle/momentum/class_2/event/px_bias": 0.099261574447155, "metrics/particle/momentum/class_2/event/py_bias": -0.42388927936553955, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.965989112854004, "metrics/particle/momentum/class_2/event/pz_bias": -1.063814640045166, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.623842239379883, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0341164730489254, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1454317569732666, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1438971906900406, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.023682095110416412, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18483395874500275, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001258426927961409, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24323271214962006, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.041216351091861725, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.898226797580719, "metrics/particle/momentum/class_3/accepted/samples": 11125.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36138540506362915, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5237420797348022, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5451736450195312, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3880289793014526, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18125812709331512, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.530759811401367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009925388731062412, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6447414755821228, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06064463034272194, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4881579875946045, "metrics/particle/momentum/class_3/assigned/samples": 14609.0, "metrics/particle/momentum/class_3/efficiency": 0.7218401432037354, "metrics/particle/momentum/class_3/event/energy_bias": 3.900454044342041, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.687944412231445, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.494279384613037, "metrics/particle/momentum/class_3/event/px_bias": -0.2575399875640869, "metrics/particle/momentum/class_3/event/py_bias": 0.4657282829284668, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.071650505065918, "metrics/particle/momentum/class_3/event/pz_bias": -1.5707398653030396, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.543196678161621, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017461059615015984, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.14881837368011475, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.08592776954174042, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.04071082919836044, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.13753244280815125, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3759838938713074, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.07237855345010757, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0521103143692017, "metrics/particle/momentum/class_4/accepted/samples": 320.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13325001299381256, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9159455299377441, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.851147174835205, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8037166595458984, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.09944751113653183, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.211111068725586, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4826595187187195, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.004485858604311943, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.401549220085144, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.8839778900146484, "metrics/particle/momentum/class_4/event/energy_bias": -0.4370247721672058, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1517324447631836, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4000052213668823, "metrics/particle/momentum/class_4/event/px_bias": 0.1266511231660843, "metrics/particle/momentum/class_4/event/py_bias": -0.015124689787626266, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3805392980575562, "metrics/particle/momentum/class_4/event/pz_bias": 0.5214406251907349, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.0493645668029785, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011422613635659218, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.030284851789474487, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026210833340883255, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003978358116000891, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07508724927902222, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.23340022563934326, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.020724110305309296, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9094752073287964, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033592868596315384, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05939366668462753, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.055445075035095215, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01568525657057762, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.22577209770679474, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6496099829673767, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41694414615631104, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3012800216674805, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.553110659122467, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.4360859394073486, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9659708142280579, "metrics/particle/momentum/class_5/event/px_bias": 0.11267820000648499, "metrics/particle/momentum/class_5/event/py_bias": 0.13818198442459106, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8827019333839417, "metrics/particle/momentum/class_5/event/pz_bias": -0.2119462937116623, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.901387929916382, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.1736204624176025, "step_valid_time": 2732.2004375457764, "step_total_time": 2733.3740582466125, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9965826293895399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08243086591347293, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3014, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2870, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2861, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9519071310116086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9522229595222296, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9520650190744734, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047777040477770406, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9489220563847429, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9492368944923689, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490794493282468, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9519071310116086, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9938645314956271, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0906794307377462, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2088, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9307589880159787, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9386750223813787, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9347002451526633, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06132497761862131, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9267643142476698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346463742166518, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9306886561176733, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9307589880159787, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9908313373211973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09194249403788624, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1150, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8949416342412452, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9005481597494127, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.897736143637783, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09945184025058731, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.890272373540856, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8958496476115897, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8930523028883685, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8949416342412452} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_15000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..f7fc348b70a9dd3408bcbd10b4efab36ff53354b --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7106310403638431, "train": {"Classification_binary": 2.7865419387817383, "Classification": 0.024294883012771606, "Regression_pt": 0.12613485753536224, "Regression_eta": 0.010344767943024635, "Regression_sin_phi": 0.0069257463328540325, "Regression_cos_phi": 0.007297741714864969, "Regression_energy": 0.12708646059036255, "Total": 8.54575252532959, "Total_unweighted": 3.0886263959109783}, "train_group": {}, "valid": {"Classification_binary": 2.8846829732259116, "Classification": 0.0237215260664622, "Regression_pt": 0.15270410974820456, "Regression_eta": 0.01190001517534256, "Regression_sin_phi": 0.009184237569570541, "Regression_cos_phi": 0.009092271327972412, "Regression_energy": 0.1559797724088033, "Total": 9.743534723917643, "Total_unweighted": 3.2472649055222673, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.024599507451057434, "diagnostic/loss/clic_pf/Classification_binary": 2.8956072330474854, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008852967992424965, "diagnostic/loss/clic_pf/Regression_energy": 0.1590418964624405, "diagnostic/loss/clic_pf/Regression_eta": 0.011232594028115273, "diagnostic/loss/clic_pf/Regression_pt": 0.15603017807006836, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008885442279279232, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05384788289666176, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20693077147006989, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06940150260925293, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4640583097934723, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.007812430150806904, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20229922235012054, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07665695995092392, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4673773944377899, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9575009346008301, "metrics/particle/class_1/pid_accuracy": 0.9867342114448547, "metrics/particle/class_2/efficiency": 0.39738383889198303, "metrics/particle/class_2/pid_accuracy": 0.6983541250228882, "metrics/particle/class_3/efficiency": 0.721651017665863, "metrics/particle/class_3/pid_accuracy": 0.9820495247840881, "metrics/particle/class_4/efficiency": 0.9176356792449951, "metrics/particle/class_4/pid_accuracy": 0.304118275642395, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.523421585559845, "metrics/particle/count/bias_mean": 0.1437908560037613, "metrics/particle/count/mae": 3.461437940597534, "metrics/particle/count/prediction_mean": 70.14313507080078, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06606654822826385, "metrics/particle/event/energy_response_mean": 1.0339317321777344, "metrics/particle/event/met_abs_error": 5.746825218200684, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05077304691076279, "metrics/particle/event/scalar_pt_response_mean": 1.0275379419326782, "metrics/particle/event/vector_pt_closure": 0.05081160366535187, "metrics/particle/matched/delta_eta_abs_mean": 0.014533272944390774, "metrics/particle/matched/delta_phi_abs_mean": 0.012640980072319508, "metrics/particle/matched/delta_r_mean": 0.02153376303613186, "metrics/particle/matched/energy_relative_abs_mean": 0.08738712966442108, "metrics/particle/matched/pid_accuracy": 0.9630997776985168, "metrics/particle/matched/pt_relative_abs_mean": 0.07839439064264297, "metrics/particle/matching/duplicate_fraction": 0.03229623660445213, "metrics/particle/matching/efficiency": 0.8087096810340881, "metrics/particle/matching/f1": 0.8078799247741699, "metrics/particle/matching/purity": 0.8070518970489502, "metrics/particle/matching/target_coverage_dr0p05": 0.7467296719551086, "metrics/particle/matching/target_coverage_dr0p10": 0.8429023623466492, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7183260321617126, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8537801504135132, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.026763737201690674, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08439315855503082, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07796274870634079, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017289867624640465, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13336332142353058, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003038405440747738, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3510472774505615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05440770834684372, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2142857313156128, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8228.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33162400126457214, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1872211694717407, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.4473539590835571, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.3243930339813232, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1476103514432907, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 22.718847274780273, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016540052369236946, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.912136435508728, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04189884290099144, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.026933670043945, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10399.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.765466570854187, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.6847271919250488, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.285879135131836, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.7164909839630127, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09185725450515747, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.28491365909576416, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.0679450035095215, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.25390350818634033, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.061360359191895, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03668120130896568, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09524029493331909, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09092587977647781, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024048928171396255, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1482507288455963, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019643953070044518, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9652359485626221, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06451289355754852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1720855236053467, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1629.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8415434956550598, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7878159284591675, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11500.1044921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11499.943359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3014320731163025, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 615148.5625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08487600088119507, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.5045809745788574, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.00922815129160881, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.063502311706543, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2863.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5302734375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 2.526944160461426, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.827625274658203, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4860382080078125, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05388807877898216, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.020293734967708588, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.61591911315918, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.901298463344574, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.91122817993164, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018023720011115074, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0879862830042839, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07706396281719208, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013078209012746811, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13311733305454254, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006311139441095293, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11407959461212158, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004243513569235802, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5761880874633789, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19014.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17147347331047058, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9601212739944458, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6698776483535767, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5595781803131104, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09355807304382324, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.234105110168457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034172246232628822, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2920282781124115, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002271260367706418, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8363085985183716, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21655.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8660836219787598, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.9797489643096924, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.027888298034668, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.543520927429199, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9671471118927002, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.43965256214141846, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.906661033630371, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3923023045063019, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.585638046264648, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021567272022366524, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08737156540155411, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07810225337743759, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014897502958774567, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13408640027046204, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023899413645267487, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22963857650756836, "metrics/particle/momentum/all/accepted/pz_bias": 0.021940505132079124, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0984545946121216, "metrics/particle/momentum/all/accepted/samples": 28871.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27411162853240967, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0956225395202637, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 943.791259765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 943.6729125976562, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12670046091079712, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 176145.78125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014004639349877834, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7401206493377686, "metrics/particle/momentum/all/assigned/pz_bias": 0.013130282051861286, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.117888450622559, "metrics/particle/momentum/all/assigned/samples": 34917.0, "metrics/particle/momentum/all/efficiency": 0.8070160746574402, "metrics/particle/momentum/all/event/energy_bias": 8.191420555114746, "metrics/particle/momentum/all/event/p3_error_mean": 17.344093322753906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.292051315307617, "metrics/particle/momentum/all/event/px_bias": -0.821401834487915, "metrics/particle/momentum/all/event/py_bias": 0.7042725086212158, "metrics/particle/momentum/all/event/pz_abs_mean": 13.24902629852295, "metrics/particle/momentum/all/event/pz_bias": 0.7628996968269348, "metrics/particle/momentum/all/event/pz_residual_rms": 21.75101089477539, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01179248932749033, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04062666743993759, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.025939498096704483, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.01199179794639349, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06618581712245941, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0008068520110100508, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11714310944080353, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.019035661593079567, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7632919549942017, "metrics/particle/momentum/class_1/accepted/samples": 16112.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06240006908774376, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2780502140522003, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2808099687099457, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.25261837244033813, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02909974567592144, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.67132568359375, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004800568800419569, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29813364148139954, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.023960191756486893, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.715599536895752, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9548983573913574, "metrics/particle/momentum/class_1/event/energy_bias": 7.501701354980469, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.335883140563965, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.901123523712158, "metrics/particle/momentum/class_1/event/px_bias": -0.8997697830200195, "metrics/particle/momentum/class_1/event/py_bias": -0.5159537196159363, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.438450813293457, "metrics/particle/momentum/class_1/event/pz_bias": 0.19524945318698883, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.32936668395996, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043447595089673996, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.204472616314888, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.201168954372406, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04488130286335945, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24237479269504547, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03637901693582535, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.69717538356781, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04339316859841347, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6460132598876953, "metrics/particle/momentum/class_2/accepted/samples": 1182.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0056203603744507, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.001920223236084, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12205.3486328125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12204.8310546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40304040908813477, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 633797.0625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08750463277101517, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.677769899368286, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.02955160103738308, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.814406394958496, "metrics/particle/momentum/class_2/assigned/samples": 2697.0, "metrics/particle/momentum/class_2/efficiency": 0.40122199058532715, "metrics/particle/momentum/class_2/event/energy_bias": -5.304437637329102, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.850440979003906, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.031017303466797, "metrics/particle/momentum/class_2/event/px_bias": 0.19906288385391235, "metrics/particle/momentum/class_2/event/py_bias": -0.06622388958930969, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.779306411743164, "metrics/particle/momentum/class_2/event/pz_bias": -0.6931078433990479, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.488466262817383, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03384269028902054, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14242404699325562, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1422644853591919, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.025414038449525833, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1839466392993927, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011747695971280336, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23362824320793152, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02356836572289467, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9130812287330627, "metrics/particle/momentum/class_3/accepted/samples": 11066.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3885425925254822, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0282301902770996, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0984904766082764, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9473024606704712, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19056947529315948, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.4694881439209, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011484158225357533, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7180289626121521, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003201306564733386, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.699667453765869, "metrics/particle/momentum/class_3/assigned/samples": 14803.0, "metrics/particle/momentum/class_3/efficiency": 0.7180119156837463, "metrics/particle/momentum/class_3/event/energy_bias": 7.703716278076172, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.735015869140625, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.764906883239746, "metrics/particle/momentum/class_3/event/px_bias": -0.4102809429168701, "metrics/particle/momentum/class_3/event/py_bias": 1.1044715642929077, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.937021255493164, "metrics/particle/momentum/class_3/event/pz_bias": 0.9094053506851196, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.772120475769043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014838503673672676, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.13559338450431824, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0613456629216671, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.021545493975281715, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10740412026643753, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3273358643054962, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08926432579755783, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0335105657577515, "metrics/particle/momentum/class_4/accepted/samples": 331.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13127411901950836, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.296630620956421, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0013279914855957, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9564266204833984, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0635359138250351, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.510116577148438, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4277574419975281, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.024490371346473694, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3311930894851685, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9143646359443665, "metrics/particle/momentum/class_4/event/energy_bias": -0.6104940176010132, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.176647424697876, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3834115266799927, "metrics/particle/momentum/class_4/event/px_bias": 0.09493765234947205, "metrics/particle/momentum/class_4/event/py_bias": 0.09351120889186859, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.428985595703125, "metrics/particle/momentum/class_4/event/pz_bias": 0.6688238978385925, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.130638599395752, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010546251200139523, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.029417650774121284, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025371678173542023, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008779485709965229, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06699631363153458, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.23747920989990234, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08279458433389664, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9322828650474548, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0384783074259758, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.037715863436460495, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.032781437039375305, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009488820098340511, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09938637912273407, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6022450923919678, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.44926032423973083, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.037276744842529, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -1.099066138267517, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5066030025482178, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.0545861721038818, "metrics/particle/momentum/class_5/event/px_bias": 0.19464834034442902, "metrics/particle/momentum/class_5/event/py_bias": 0.08846734464168549, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8770438432693481, "metrics/particle/momentum/class_5/event/pz_bias": -0.3174712061882019, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.8763935565948486, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.1392078399658203, "step_valid_time": 2901.8002450466156, "step_total_time": 2902.9394533634186, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0123744506084662, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08121375075515525, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2998, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2833, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2823, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9396351575456053, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9449633088725817, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.94229170131382, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.055036691127418276, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.936318407960199, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9416277518345564, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9389655745883918, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9396351575456053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0116692757718626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09942277307633174, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2112, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400798934753661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9421708185053381, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941124194623417, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05782918149466192, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9374167776298269, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9395017793594306, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.938458120417685, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400798934753661, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009690447454585, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10357387090605152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1150, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8949416342412452, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8942457231726283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8945935433683392, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1057542768273717, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.890272373540856, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8895800933125972, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8899260987942434, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8949416342412452} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_16000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..3176d6e087d8523f2c9a096f01568a41a433f668 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.7580064430547659, "train": {"Classification_binary": 2.8360650539398193, "Classification": 0.023536566644906998, "Regression_pt": 0.1489950567483902, "Regression_eta": 0.012967116199433804, "Regression_sin_phi": 0.009973675012588501, "Regression_cos_phi": 0.012442036531865597, "Regression_energy": 0.1481800526380539, "Total": 9.818805694580078, "Total_unweighted": 3.1921595577150583}, "train_group": {}, "valid": {"Classification_binary": 2.8379761377970376, "Classification": 0.023255884647369385, "Regression_pt": 0.149822731812795, "Regression_eta": 0.01175236701965332, "Regression_sin_phi": 0.009058546274900436, "Regression_cos_phi": 0.009080483267704645, "Regression_energy": 0.1526537537574768, "Total": 9.577323277791342, "Total_unweighted": 3.193599904576937, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.023869715631008148, "diagnostic/loss/clic_pf/Classification_binary": 2.8452720642089844, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00880757998675108, "diagnostic/loss/clic_pf/Regression_energy": 0.15691393613815308, "diagnostic/loss/clic_pf/Regression_eta": 0.011233046650886536, "diagnostic/loss/clic_pf/Regression_pt": 0.15441837906837463, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008812746964395046, "diagnostic/regression/clic_pf/energy_pred_mean": 0.042422350496053696, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20086491107940674, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05797597020864487, "diagnostic/regression/clic_pf/energy_residual_rms": 0.457021027803421, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0015106915961951017, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19951175153255463, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07035522162914276, "diagnostic/regression/clic_pf/pt_residual_rms": 0.46368512511253357, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9607410430908203, "metrics/particle/class_1/pid_accuracy": 0.9869444966316223, "metrics/particle/class_2/efficiency": 0.39715832471847534, "metrics/particle/class_2/pid_accuracy": 0.7197614908218384, "metrics/particle/class_3/efficiency": 0.7270610928535461, "metrics/particle/class_3/pid_accuracy": 0.981886625289917, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.32116788625717163, "metrics/particle/class_5/efficiency": 0.9959758520126343, "metrics/particle/class_5/pid_accuracy": 0.6282828450202942, "metrics/particle/count/bias_mean": -0.46666666865348816, "metrics/particle/count/mae": 3.538562059402466, "metrics/particle/count/prediction_mean": 69.53267669677734, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05821545049548149, "metrics/particle/event/energy_response_mean": 1.004869818687439, "metrics/particle/event/met_abs_error": 5.390781879425049, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04481123387813568, "metrics/particle/event/scalar_pt_response_mean": 1.0100990533828735, "metrics/particle/event/vector_pt_closure": 0.04662080109119415, "metrics/particle/matched/delta_eta_abs_mean": 0.01262684166431427, "metrics/particle/matched/delta_phi_abs_mean": 0.011760248802602291, "metrics/particle/matched/delta_r_mean": 0.019436554983258247, "metrics/particle/matched/energy_relative_abs_mean": 0.08211205154657364, "metrics/particle/matched/pid_accuracy": 0.9647977948188782, "metrics/particle/matched/pt_relative_abs_mean": 0.07545236498117447, "metrics/particle/matching/duplicate_fraction": 0.031047610566020012, "metrics/particle/matching/efficiency": 0.8127059936523438, "metrics/particle/matching/f1": 0.8154240846633911, "metrics/particle/matching/purity": 0.8181604743003845, "metrics/particle/matching/target_coverage_dr0p05": 0.7679250240325928, "metrics/particle/matching/target_coverage_dr0p10": 0.8464317917823792, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7231066823005676, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8541536331176758, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02253103069961071, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08160270750522614, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07623999565839767, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.007339437957853079, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.132781520485878, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003373087616637349, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35591259598731995, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 4.9487207434140146e-05, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2542566061019897, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8301.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.32452794909477234, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0086424350738525, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1385579109191895, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.010428547859192, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1394719034433365, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.1740140914917, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014604894444346428, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8149656057357788, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.016393836587667465, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8701369762420654, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10339.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7722578644752502, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.798811435699463, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.23998737335205, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6225662231445312, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.05640292167663574, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.26363620162010193, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.113167762756348, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.2832458019256592, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.086852073669434, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03121262975037098, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09546830505132675, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09285012632608414, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019967393949627876, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15018077194690704, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01752336509525776, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9540448784828186, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.045995764434337616, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0314643383026123, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1712.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8164050579071045, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5253500938415527, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 64078.0703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 64077.8984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2765353322029114, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3400810.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07632233202457428, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2588350772857666, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06574426591396332, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.266446113586426, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2817.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5572916865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.8968901634216309, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.547689437866211, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3758962154388428, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.056908804923295975, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.350004106760025, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.36337947845459, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.7270172834396362, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.454740524291992, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017072303220629692, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08140774816274643, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07365641742944717, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009297560900449753, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13020680844783783, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008418394136242568, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1204758733510971, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021379077807068825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6150415539741516, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19006.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15588079392910004, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7084712982177734, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5352729558944702, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42529526352882385, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09240312874317169, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.5739054679870605, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028702374547719955, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2653111219406128, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0222554299980402, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5437036752700806, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21601.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8657192587852478, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.45292750000953674, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.9846272468566895, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.602941513061523, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.09321948885917664, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.025970220565796, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.868342399597168, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.10798327624797821, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.886590003967285, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01946801319718361, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08229303359985352, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07552780956029892, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009366908110678196, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13220404088497162, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002550053410232067, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23700056970119476, "metrics/particle/momentum/all/accepted/pz_bias": -0.011274517513811588, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1135165691375732, "metrics/particle/momentum/all/accepted/samples": 29019.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25958192348480225, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.863968551158905, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5194.09765625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5193.9775390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12132807821035385, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 968176.875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012314066290855408, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6714346408843994, "metrics/particle/momentum/all/assigned/pz_bias": -0.0036263789515942335, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.7708280086517334, "metrics/particle/momentum/all/assigned/samples": 34757.0, "metrics/particle/momentum/all/efficiency": 0.8111530542373657, "metrics/particle/momentum/all/event/energy_bias": 0.3548489212989807, "metrics/particle/momentum/all/event/p3_error_mean": 16.660120010375977, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.138134002685547, "metrics/particle/momentum/all/event/px_bias": -0.020092252641916275, "metrics/particle/momentum/all/event/py_bias": 1.6396105289459229, "metrics/particle/momentum/all/event/pz_abs_mean": 12.516317367553711, "metrics/particle/momentum/all/event/pz_bias": -0.6642117500305176, "metrics/particle/momentum/all/event/pz_residual_rms": 21.70236587524414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010161776095628738, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03342093527317047, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022574102506041527, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00469954265281558, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06427482515573502, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015456905821338296, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13240043818950653, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.021486682817339897, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8707829713821411, "metrics/particle/momentum/class_1/accepted/samples": 16174.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05488542467355728, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1906248778104782, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22253121435642242, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.19222263991832733, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.025724615901708603, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.460753440856934, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005334598012268543, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2845187783241272, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.019222430884838104, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.517225742340088, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9585728645324707, "metrics/particle/momentum/class_1/event/energy_bias": 5.363016605377197, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.289663314819336, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.6448445320129395, "metrics/particle/momentum/class_1/event/px_bias": -0.8138467073440552, "metrics/particle/momentum/class_1/event/py_bias": -0.2970075011253357, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.576862335205078, "metrics/particle/momentum/class_1/event/pz_bias": -0.9456101059913635, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.654308319091797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04114101082086563, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2003689855337143, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19866248965263367, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.046758104115724564, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23736411333084106, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029259895905852318, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6261578798294067, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04294411465525627, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4106063842773438, "metrics/particle/momentum/class_2/accepted/samples": 1162.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9603678584098816, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7185398936271667, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 68371.90625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 68371.375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3848484754562378, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3512965.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07348484545946121, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2354207038879395, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.21602769196033478, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.122385025024414, "metrics/particle/momentum/class_2/assigned/samples": 2640.0, "metrics/particle/momentum/class_2/efficiency": 0.3944331407546997, "metrics/particle/momentum/class_2/event/energy_bias": -9.7075834274292, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.073461532592773, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.065180778503418, "metrics/particle/momentum/class_2/event/px_bias": 0.6382289528846741, "metrics/particle/momentum/class_2/event/py_bias": 1.2184324264526367, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.95229721069336, "metrics/particle/momentum/class_2/event/pz_bias": -0.18581241369247437, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.27165985107422, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03100944682955742, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14098578691482544, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1409166306257248, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02210952900350094, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1822196990251541, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013428827514871955, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2428760826587677, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0005982439033687115, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9383013844490051, "metrics/particle/momentum/class_3/accepted/samples": 11170.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3744603097438812, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6738849878311157, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7310919761657715, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5785341262817383, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1865052431821823, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.144133567810059, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009658549912273884, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6641247868537903, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.017877962440252304, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.7827320098876953, "metrics/particle/momentum/class_3/assigned/samples": 14702.0, "metrics/particle/momentum/class_3/efficiency": 0.7247599363327026, "metrics/particle/momentum/class_3/event/energy_bias": 5.677097320556641, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.101447105407715, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.6385602951049805, "metrics/particle/momentum/class_3/event/px_bias": -0.1382816880941391, "metrics/particle/momentum/class_3/event/py_bias": 0.5594147443771362, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.3330659866333, "metrics/particle/momentum/class_3/event/pz_bias": 0.36948761343955994, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.859269142150879, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014025870710611343, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1068015843629837, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.054135143756866455, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0070189512334764, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09993728995323181, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2653621435165405, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0005784882814623415, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8606357574462891, "metrics/particle/momentum/class_4/accepted/samples": 332.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.14879214763641357, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.8322286605834961, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.580894947052002, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.5304990410804749, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.042585372924805, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3875281512737274, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.11273585259914398, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2447913885116577, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9171270728111267, "metrics/particle/momentum/class_4/event/energy_bias": -0.705252468585968, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.989593267440796, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2822681665420532, "metrics/particle/momentum/class_4/event/px_bias": 0.09701256453990936, "metrics/particle/momentum/class_4/event/py_bias": 0.013788392767310143, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2699755430221558, "metrics/particle/momentum/class_4/event/pz_bias": 0.3814181089401245, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.716248989105225, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009656167589128017, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02438182197511196, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020831430330872536, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004680076614022255, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06213542819023132, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.25112560391426086, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.12528298795223236, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8153319358825684, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.009710860438644886, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.028862882405519485, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02532009221613407, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 5.1239570893812925e-05, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.08772654831409454, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5009830594062805, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3758319020271301, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.485539436340332, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.27242934703826904, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.4133217334747314, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9406347274780273, "metrics/particle/momentum/class_5/event/px_bias": 0.19679464399814606, "metrics/particle/momentum/class_5/event/py_bias": 0.14498256146907806, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8770402669906616, "metrics/particle/momentum/class_5/event/pz_bias": -0.2836949825286865, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.9323530197143555, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.233311653137207, "step_valid_time": 3068.611474275589, "step_total_time": 3069.844786167145, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032636697600814, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07447388007330635, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2863, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2857, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9495854063018242, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517952127659575, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9506890254026232, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048204787234042555, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9475953565505805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9498005319148937, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9486966627926282, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9495854063018242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0037645471826826, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08880935042677507, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.933422103861518, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9388392857142858, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936122857778767, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061160714285714284, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9307589880159787, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9361607142857142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9334520365012241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.933422103861518, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066645374817604, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0950560432004357, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1262, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1160, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9027237354085603, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.919175911251981, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9108755398508048, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08082408874801902, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8964980544747082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9128367670364501, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9045936395759717, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9027237354085603} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_17000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..9f026c012778d47c3cf49df78e2107b236c4bce1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8053818457456888, "train": {"Classification_binary": 2.8548450469970703, "Classification": 0.022113662213087082, "Regression_pt": 0.172406867146492, "Regression_eta": 0.012186417356133461, "Regression_sin_phi": 0.00920434296131134, "Regression_cos_phi": 0.010183155536651611, "Regression_energy": 0.16783247888088226, "Total": 10.179752349853516, "Total_unweighted": 3.248771971091628}, "train_group": {}, "valid": {"Classification_binary": 2.827239672342936, "Classification": 0.022992307941118877, "Regression_pt": 0.1480963925520579, "Regression_eta": 0.01164684072136879, "Regression_sin_phi": 0.009052764003475508, "Regression_cos_phi": 0.008980963379144669, "Regression_energy": 0.15029477079709372, "Total": 9.490338007609049, "Total_unweighted": 3.1783037117371955, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02370164357125759, "diagnostic/loss/clic_pf/Classification_binary": 2.835338592529297, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008735869079828262, "diagnostic/loss/clic_pf/Regression_energy": 0.15363991260528564, "diagnostic/loss/clic_pf/Regression_eta": 0.010968427173793316, "diagnostic/loss/clic_pf/Regression_pt": 0.15150004625320435, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008701068349182606, "diagnostic/regression/clic_pf/energy_pred_mean": 0.037786971777677536, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2011571079492569, "diagnostic/regression/clic_pf/energy_residual_mean": 0.053340595215559006, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4556668996810913, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0017144489102065563, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20136265456676483, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07055898010730743, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4626046121120453, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9615560173988342, "metrics/particle/class_1/pid_accuracy": 0.9859012365341187, "metrics/particle/class_2/efficiency": 0.41384753584861755, "metrics/particle/class_2/pid_accuracy": 0.7062670588493347, "metrics/particle/class_3/efficiency": 0.7262420654296875, "metrics/particle/class_3/pid_accuracy": 0.9825488328933716, "metrics/particle/class_4/efficiency": 0.9215116500854492, "metrics/particle/class_4/pid_accuracy": 0.29652997851371765, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6232179403305054, "metrics/particle/count/bias_mean": -0.05882352963089943, "metrics/particle/count/mae": 3.430065393447876, "metrics/particle/count/prediction_mean": 69.94052124023438, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0549258328974247, "metrics/particle/event/energy_response_mean": 1.0035080909729004, "metrics/particle/event/met_abs_error": 5.444995403289795, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04397793859243393, "metrics/particle/event/scalar_pt_response_mean": 1.0093088150024414, "metrics/particle/event/vector_pt_closure": 0.04718908295035362, "metrics/particle/matched/delta_eta_abs_mean": 0.012322289869189262, "metrics/particle/matched/delta_phi_abs_mean": 0.011352261528372765, "metrics/particle/matched/delta_r_mean": 0.0188558679074049, "metrics/particle/matched/energy_relative_abs_mean": 0.08341606706380844, "metrics/particle/matched/pid_accuracy": 0.9632709622383118, "metrics/particle/matched/pt_relative_abs_mean": 0.07906832545995712, "metrics/particle/matching/duplicate_fraction": 0.03352988883852959, "metrics/particle/matching/efficiency": 0.8140038847923279, "metrics/particle/matching/f1": 0.8143460154533386, "metrics/particle/matching/purity": 0.814688503742218, "metrics/particle/matching/target_coverage_dr0p05": 0.7606326937675476, "metrics/particle/matching/target_coverage_dr0p10": 0.8482152223587036, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.719521164894104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8563292026519775, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022543076425790787, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08179990947246552, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07984567433595657, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010590066201984882, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13615909218788147, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0038314175326377153, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.370329350233078, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.018538352102041245, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2922570705413818, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8352.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3160824775695801, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0358548164367676, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.259932279586792, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1333966255187988, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13671274483203888, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 21.024187088012695, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014592934399843216, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8114517331123352, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.09436347335577011, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8094253540039062, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10416.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7770025134086609, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.163200855255127, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.698144912719727, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.450284004211426, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.004140365868806839, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.194088414311409, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.586848735809326, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.9079451560974121, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.399513244628906, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030755525454878807, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09491778910160065, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0947156473994255, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017045510932803154, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1544272005558014, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016412660479545593, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9767053127288818, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11403261870145798, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4611098766326904, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1706.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7983905076980591, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.2550625801086426, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 70612.9140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 70612.7421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2860635817050934, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3778005.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07614390552043915, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0496044158935547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.13247907161712646, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.416824340820312, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2863.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5553385615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1061632633209229, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.873439788818359, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2788457870483398, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 2.8653070330619812e-05, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.029657159000635147, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.70053768157959, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4112447500228882, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.594755172729492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016201576218008995, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08337552100419998, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07743797451257706, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0036512501537799835, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13257448375225067, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008425930864177644, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12483751028776169, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.016128109768033028, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6148281097412109, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18989.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16180482506752014, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7626959681510925, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5691639184951782, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.45532912015914917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0926535353064537, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.456547737121582, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003661645343527198, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2911854684352875, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.010265517979860306, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7464848756790161, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21575.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8649448752403259, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.2456735372543335, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.827856540679932, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.453349590301514, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3653227388858795, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.10426545143127441, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8493423461914062, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.7352325320243835, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.160593032836914, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018879761919379234, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08360039442777634, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07914503663778305, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.006433072965592146, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13498641550540924, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002616449259221554, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2454569786787033, "metrics/particle/momentum/all/accepted/pz_bias": 0.001484328182414174, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1961828470230103, "metrics/particle/momentum/all/accepted/samples": 29047.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26020100712776184, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9669155478477478, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5801.06103515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5800.9384765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12170769274234772, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1082796.75, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012882309034466743, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6732490658760071, "metrics/particle/momentum/all/assigned/pz_bias": 0.0327279232442379, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3957324028015137, "metrics/particle/momentum/all/assigned/samples": 34854.0, "metrics/particle/momentum/all/efficiency": 0.8119357228279114, "metrics/particle/momentum/all/event/energy_bias": -1.0236905813217163, "metrics/particle/momentum/all/event/p3_error_mean": 16.238378524780273, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.982573986053467, "metrics/particle/momentum/all/event/px_bias": 0.36121100187301636, "metrics/particle/momentum/all/event/py_bias": -0.26869672536849976, "metrics/particle/momentum/all/event/pz_abs_mean": 12.233331680297852, "metrics/particle/momentum/all/event/pz_bias": 2.054422378540039, "metrics/particle/momentum/all/event/pz_residual_rms": 22.84416961669922, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008926838636398315, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03490123897790909, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.025706103071570396, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.004619895946234465, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06406856328248978, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012977381702512503, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1283310055732727, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007474537473171949, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7928553819656372, "metrics/particle/momentum/class_1/accepted/samples": 16182.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.055991653352975845, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16526144742965698, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.19591134786605835, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15310728549957275, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02530820295214653, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.6900129318237305, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005275011993944645, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2876826524734497, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.038848284631967545, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.423007011413574, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9590470194816589, "metrics/particle/momentum/class_1/event/energy_bias": 2.1533870697021484, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.449243545532227, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.232476234436035, "metrics/particle/momentum/class_1/event/px_bias": -0.28228759765625, "metrics/particle/momentum/class_1/event/py_bias": -0.6145296096801758, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.863907814025879, "metrics/particle/momentum/class_1/event/pz_bias": 1.5060408115386963, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.41779899597168, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04197867214679718, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2045547068119049, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20332574844360352, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05115707963705063, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24306033551692963, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032786883413791656, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7500203847885132, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.06451958417892456, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7634270191192627, "metrics/particle/momentum/class_2/accepted/samples": 1220.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9581209421157837, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7791649103164673, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 75655.859375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 75655.34375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3753742575645447, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3910704.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08270958065986633, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.436304807662964, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04672231525182724, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.812795639038086, "metrics/particle/momentum/class_2/assigned/samples": 2672.0, "metrics/particle/momentum/class_2/efficiency": 0.4141208529472351, "metrics/particle/momentum/class_2/event/energy_bias": -8.999184608459473, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.965553283691406, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.689321517944336, "metrics/particle/momentum/class_2/event/px_bias": -0.012083113193511963, "metrics/particle/momentum/class_2/event/py_bias": -0.3187178373336792, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.113299369812012, "metrics/particle/momentum/class_2/event/pz_bias": -0.2176617681980133, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.94970703125, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.031175993382930756, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1413305699825287, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14455609023571014, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.028925981372594833, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18607141077518463, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013473457656800747, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24927914142608643, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00012204804079374298, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1081829071044922, "metrics/particle/momentum/class_3/accepted/samples": 11133.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3741121292114258, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9348196983337402, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3182966709136963, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.1719870567321777, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1888798624277115, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 62.120880126953125, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009345794096589088, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6236099600791931, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029512541368603706, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.504096508026123, "metrics/particle/momentum/class_3/assigned/samples": 14766.0, "metrics/particle/momentum/class_3/efficiency": 0.7223591804504395, "metrics/particle/momentum/class_3/event/energy_bias": 7.282994270324707, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.65751838684082, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.859462738037109, "metrics/particle/momentum/class_3/event/px_bias": 0.37647634744644165, "metrics/particle/momentum/class_3/event/py_bias": 0.3932097256183624, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.792404174804688, "metrics/particle/momentum/class_3/event/pz_bias": 0.5596584677696228, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.542409896850586, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012671280652284622, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1053953692317009, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06132670119404793, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.008590544573962688, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11345567554235458, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2910115122795105, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04249757528305054, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9628891348838806, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10674796998500824, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.7006053328514099, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.5172525644302368, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.45795562863349915, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.4556496143341064, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3373204469680786, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.01994120143353939, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0172325372695923, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.8618958592414856, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.047778367996216, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3148800134658813, "metrics/particle/momentum/class_4/event/px_bias": 0.1501232385635376, "metrics/particle/momentum/class_4/event/py_bias": 0.031560927629470825, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3241322040557861, "metrics/particle/momentum/class_4/event/pz_bias": 0.3267054855823517, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.808957576751709, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007991553284227848, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03062564507126808, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028645077720284462, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.00481199799105525, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07609212398529053, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2568760812282562, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08175551891326904, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8505926132202148, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.008095473982393742, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0411289818584919, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03917202726006508, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.008721509017050266, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11566971987485886, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5467112064361572, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3744772672653198, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6009202003479004, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.5989918112754822, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.2283978462219238, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8696123361587524, "metrics/particle/momentum/class_5/event/px_bias": 0.12898214161396027, "metrics/particle/momentum/class_5/event/py_bias": 0.23978009819984436, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.708537220954895, "metrics/particle/momentum/class_5/event/pz_bias": -0.12032045423984528, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5309953689575195, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3243567943572998, "step_valid_time": 3232.683291912079, "step_total_time": 3234.0076491832733, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9973498005837741, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07746227660833171, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3002, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2847, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9442786069651742, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9483677548301133, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.946318763503407, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051632245169886744, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9406301824212272, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447035309793471, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9426624563736081, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9442786069651742, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979907661168254, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09180983768514905, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2101, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2095, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9325343985796716, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392042914617792, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9358574610244988, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06079570853822083, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9298712827341322, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9365221278497988, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.933184855233853, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9325343985796716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979694275217645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09840225350263843, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1158, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1153, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9011673151750973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9025720966484801, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9018691588785047, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09742790335151988, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8972762645914397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8986749805144193, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8979750778816199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9011673151750973} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_18000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..4d4e45dbad3e9f9d9abe374a33cd0fa88806e4fa --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.8527572484366117, "train": {"Classification_binary": 2.647911310195923, "Classification": 0.020630715414881706, "Regression_pt": 0.13669085502624512, "Regression_eta": 0.009216725826263428, "Regression_sin_phi": 0.009206898510456085, "Regression_cos_phi": 0.010001265443861485, "Regression_energy": 0.13772618770599365, "Total": 8.86066722869873, "Total_unweighted": 2.9713839581236243}, "train_group": {}, "valid": {"Classification_binary": 2.8202975591023765, "Classification": 0.023030708233515423, "Regression_pt": 0.1464844048023224, "Regression_eta": 0.011528516809145609, "Regression_sin_phi": 0.008958957468469938, "Regression_cos_phi": 0.009002997850378355, "Regression_energy": 0.14913361271222433, "Total": 9.434308369954428, "Total_unweighted": 3.1684367569784326, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.023616325110197067, "diagnostic/loss/clic_pf/Classification_binary": 2.8202452659606934, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008706850931048393, "diagnostic/loss/clic_pf/Regression_energy": 0.15164414048194885, "diagnostic/loss/clic_pf/Regression_eta": 0.010698746889829636, "diagnostic/loss/clic_pf/Regression_pt": 0.14911729097366333, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008719237521290779, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05640178918838501, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20337651669979095, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07195542007684708, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4511024057865143, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.009248786605894566, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19895242154598236, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07809332013130188, "diagnostic/regression/clic_pf/pt_residual_rms": 0.45661473274230957, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.959846556186676, "metrics/particle/class_1/pid_accuracy": 0.9863110184669495, "metrics/particle/class_2/efficiency": 0.3962562084197998, "metrics/particle/class_2/pid_accuracy": 0.7020489573478699, "metrics/particle/class_3/efficiency": 0.7228580713272095, "metrics/particle/class_3/pid_accuracy": 0.9828547835350037, "metrics/particle/class_4/efficiency": 0.9234496355056763, "metrics/particle/class_4/pid_accuracy": 0.32423925399780273, "metrics/particle/class_5/efficiency": 0.9778671860694885, "metrics/particle/class_5/pid_accuracy": 0.6666666865348816, "metrics/particle/count/bias_mean": -1.3287581205368042, "metrics/particle/count/mae": 3.6124184131622314, "metrics/particle/count/prediction_mean": 68.67058563232422, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06038772314786911, "metrics/particle/event/energy_response_mean": 1.0194374322891235, "metrics/particle/event/met_abs_error": 5.342158794403076, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04558363929390907, "metrics/particle/event/scalar_pt_response_mean": 1.012283444404602, "metrics/particle/event/vector_pt_closure": 0.04791475087404251, "metrics/particle/matched/delta_eta_abs_mean": 0.01084796991199255, "metrics/particle/matched/delta_phi_abs_mean": 0.010948843322694302, "metrics/particle/matched/delta_r_mean": 0.017324382439255714, "metrics/particle/matched/energy_relative_abs_mean": 0.08799866586923599, "metrics/particle/matched/pid_accuracy": 0.9644030332565308, "metrics/particle/matched/pt_relative_abs_mean": 0.07786805182695389, "metrics/particle/matching/duplicate_fraction": 0.03004777990281582, "metrics/particle/matching/efficiency": 0.8102503418922424, "metrics/particle/matching/f1": 0.8180142641067505, "metrics/particle/matching/purity": 0.8259284496307373, "metrics/particle/matching/target_coverage_dr0p05": 0.7767579555511475, "metrics/particle/matching/target_coverage_dr0p10": 0.8445830345153809, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7174670100212097, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8493916988372803, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02082669548690319, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08664117008447647, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07793737947940826, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01607642136514187, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13268086314201355, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0036231884732842445, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3508954644203186, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.013200235553085804, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2365124225616455, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8280.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.31769856810569763, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9942360520362854, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8929263949394226, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7708476781845093, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13549897074699402, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.832856178283691, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015218027867376804, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8125666975975037, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.016363879665732384, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9349167346954346, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10251.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7703042030334473, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5988906621932983, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.543644428253174, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3847544193267822, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.16937291622161865, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.22672878205776215, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.4620442390441895, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4198235869407654, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.535319328308105, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028312666341662407, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10099303722381592, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09598205983638763, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019917679950594902, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15503942966461182, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01731102168560028, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9252406358718872, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0022715451195836067, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8826887607574463, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1733.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7716875076293945, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.359269142150879, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12445.984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12445.8173828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26983559131622314, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 658080.0625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08398856222629547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.192349910736084, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.033595480024814606, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.667638778686523, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2798.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5641276240348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.5738039016723633, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.644498825073242, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3754106760025024, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02561817690730095, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.06015993654727936, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.468130111694336, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.1302976608276367, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.24029541015625, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014838325791060925, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08787721395492554, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0765373483300209, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017710821703076363, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13265037536621094, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005814260803163052, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1174040138721466, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00962568074464798, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.613666296005249, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18919.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15252813696861267, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.786942720413208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6112268567085266, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5121867060661316, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09040074795484543, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.64732837677002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031220875680446625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2724027633666992, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.008600899949669838, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6728192567825317, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21460.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8617563843727112, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.281045138835907, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.3283891677856445, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.0056915283203125, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3771379888057709, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.16859504580497742, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6451714038848877, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.167920470237732, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.223756790161133, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017359228804707527, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08830910176038742, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07810274511575699, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017375262454152107, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.134105384349823, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002454030094668269, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23261520266532898, "metrics/particle/momentum/all/accepted/pz_bias": 0.0026526644360274076, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.087005376815796, "metrics/particle/momentum/all/accepted/samples": 28932.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25179412961006165, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.976004421710968, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1009.769287109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1009.6580200195312, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11834593862295151, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 187385.6875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01327189989387989, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6696102023124695, "metrics/particle/momentum/all/assigned/pz_bias": 0.012933483347296715, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4462947845458984, "metrics/particle/momentum/all/assigned/samples": 34509.0, "metrics/particle/momentum/all/efficiency": 0.8087211847305298, "metrics/particle/momentum/all/event/energy_bias": 2.453739643096924, "metrics/particle/momentum/all/event/p3_error_mean": 16.489112854003906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.60612154006958, "metrics/particle/momentum/all/event/px_bias": -0.5721290707588196, "metrics/particle/momentum/all/event/py_bias": 0.3351638913154602, "metrics/particle/momentum/all/event/pz_abs_mean": 12.74081802368164, "metrics/particle/momentum/all/event/pz_bias": -0.542553722858429, "metrics/particle/momentum/all/event/pz_residual_rms": 21.44106101989746, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008348595350980759, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04183492064476013, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02581382356584072, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00970670860260725, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06589488685131073, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00148524041287601, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12636129558086395, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0013912675203755498, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.812832236289978, "metrics/particle/momentum/class_1/accepted/samples": 16159.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05768279731273651, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.15261133015155792, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.13993901014328003, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10849550366401672, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02566838636994362, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.356055736541748, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005809472873806953, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2788277566432953, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.021523265168070793, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2104921340942383, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.9576838612556458, "metrics/particle/momentum/class_1/event/energy_bias": 3.470989227294922, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.18596363067627, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.336408615112305, "metrics/particle/momentum/class_1/event/px_bias": -0.52704918384552, "metrics/particle/momentum/class_1/event/py_bias": -0.24942904710769653, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.584397792816162, "metrics/particle/momentum/class_1/event/pz_bias": 1.1720476150512695, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.672309875488281, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.039717331528663635, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19944311678409576, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19957441091537476, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03951132297515869, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23971547186374664, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02660944126546383, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5736037492752075, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.17371568083763123, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.346661329269409, "metrics/particle/momentum/class_2/accepted/samples": 1165.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9484572410583496, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.87518310546875, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 13519.40625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 13518.916015625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37514564394950867, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 685984.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08038835227489471, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.334477663040161, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.3324863612651825, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.479350090026855, "metrics/particle/momentum/class_2/assigned/samples": 2575.0, "metrics/particle/momentum/class_2/efficiency": 0.39545145630836487, "metrics/particle/momentum/class_2/event/energy_bias": -6.204267978668213, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.35441780090332, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.511693477630615, "metrics/particle/momentum/class_2/event/px_bias": -0.20302827656269073, "metrics/particle/momentum/class_2/event/py_bias": -0.417970210313797, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.657809257507324, "metrics/particle/momentum/class_2/event/pz_bias": -0.10012954473495483, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.209232330322266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.028421610593795776, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1444130688905716, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14285503327846527, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03486345335841179, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18464165925979614, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014415712794288993, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.244287371635437, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.010727385058999062, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9463827610015869, "metrics/particle/momentum/class_3/accepted/samples": 11099.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35991838574409485, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.925033688545227, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.101438522338867, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9625811576843262, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18290750682353973, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 63.61072540283203, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010467598214745522, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6584939956665039, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04633420705795288, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0969836711883545, "metrics/particle/momentum/class_3/assigned/samples": 14521.0, "metrics/particle/momentum/class_3/efficiency": 0.7201531529426575, "metrics/particle/momentum/class_3/event/energy_bias": 6.126499176025391, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.546998977661133, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.281296253204346, "metrics/particle/momentum/class_3/event/px_bias": -0.09055717289447784, "metrics/particle/momentum/class_3/event/py_bias": 0.8038098812103271, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.959641456604004, "metrics/particle/momentum/class_3/event/pz_bias": -1.533336877822876, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.999322891235352, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012477508746087551, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11449217796325684, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05955130234360695, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.012377660721540451, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10507751256227493, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26792284846305847, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0015703624812886119, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8951060771942139, "metrics/particle/momentum/class_4/accepted/samples": 331.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1267162710428238, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.4610910415649414, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.13537335395813, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.081347703933716, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.740922927856445, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.42376819252967834, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0669071227312088, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2687182426452637, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9143646359443665, "metrics/particle/momentum/class_4/event/energy_bias": -0.47282201051712036, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.022834300994873, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2644463777542114, "metrics/particle/momentum/class_4/event/px_bias": 0.0992436632514, "metrics/particle/momentum/class_4/event/py_bias": 0.1091003566980362, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3523774147033691, "metrics/particle/momentum/class_4/event/pz_bias": 0.04628985747694969, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.679652690887451, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008314741775393486, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03291977569460869, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026849092915654182, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004689098335802555, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08083382993936539, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.308283269405365, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1661231368780136, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.1974352598190308, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.008731644600629807, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04731467366218567, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04131608083844185, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010227875784039497, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13045574724674225, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5624061226844788, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4168533980846405, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.658520221710205, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.46665865182876587, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1846821308135986, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7683484554290771, "metrics/particle/momentum/class_5/event/px_bias": 0.14926192164421082, "metrics/particle/momentum/class_5/event/py_bias": 0.08965300768613815, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7717570066452026, "metrics/particle/momentum/class_5/event/pz_bias": -0.12742479145526886, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.779172897338867, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3389232158660889, "step_valid_time": 3395.0294229984283, "step_total_time": 3396.368346452713, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028001757019511, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07211095987607186, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3005, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2849, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9482587064676616, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.951414309484193, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498338870431894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04858569051580699, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9449419568822554, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9480865224625624, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9465116279069767, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9482587064676616, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048564011512422, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0892189627189266, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2094, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9343098091433644, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9401518535060295, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9372217275155833, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05984814649397052, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.929427430093209, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9352389459580169, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9323241317898486, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9343098091433644, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00591930471653, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08921050063600333, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1271, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1155, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1149, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8988326848249028, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9087332808811959, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.903755868544601, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09126671911880409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8941634241245137, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9040125885129819, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8990610328638498, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8988326848249028} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_19000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..82526a3e1f9da54bee40efe9edb08803c6d463dc --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.9001326511275346, "train": {"Classification_binary": 2.723372459411621, "Classification": 0.02066585049033165, "Regression_pt": 0.1357448548078537, "Regression_eta": 0.009856457822024822, "Regression_sin_phi": 0.007152467966079712, "Regression_cos_phi": 0.009050866588950157, "Regression_energy": 0.1365533322095871, "Total": 8.720194816589355, "Total_unweighted": 3.0423962892964482}, "train_group": {}, "valid": {"Classification_binary": 2.82352352142334, "Classification": 0.022799380123615265, "Regression_pt": 0.14803229769070944, "Regression_eta": 0.011302396655082703, "Regression_sin_phi": 0.008817306409279505, "Regression_cos_phi": 0.008858444169163704, "Regression_energy": 0.15073691805203757, "Total": 9.4453493754069, "Total_unweighted": 3.174070264523228, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.023512594401836395, "diagnostic/loss/clic_pf/Classification_binary": 2.8396809101104736, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008418680168688297, "diagnostic/loss/clic_pf/Regression_energy": 0.1544102430343628, "diagnostic/loss/clic_pf/Regression_eta": 0.010554822161793709, "diagnostic/loss/clic_pf/Regression_pt": 0.1521398276090622, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008357115089893341, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05509800836443901, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20132963359355927, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07065162807703018, "diagnostic/regression/clic_pf/energy_residual_rms": 0.45407405495643616, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.015199150890111923, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19952471554279327, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08404368907213211, "diagnostic/regression/clic_pf/pt_residual_rms": 0.45975351333618164, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9641203284263611, "metrics/particle/class_1/pid_accuracy": 0.9868459105491638, "metrics/particle/class_2/efficiency": 0.42298150062561035, "metrics/particle/class_2/pid_accuracy": 0.7339376211166382, "metrics/particle/class_3/efficiency": 0.7295613884925842, "metrics/particle/class_3/pid_accuracy": 0.9803237915039062, "metrics/particle/class_4/efficiency": 0.932170569896698, "metrics/particle/class_4/pid_accuracy": 0.295218288898468, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6211812496185303, "metrics/particle/count/bias_mean": 1.5908496379852295, "metrics/particle/count/mae": 3.5999999046325684, "metrics/particle/count/prediction_mean": 71.59019470214844, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06491769105195999, "metrics/particle/event/energy_response_mean": 1.0361478328704834, "metrics/particle/event/met_abs_error": 6.063751220703125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05605675280094147, "metrics/particle/event/scalar_pt_response_mean": 1.0396531820297241, "metrics/particle/event/vector_pt_closure": 0.04879043996334076, "metrics/particle/matched/delta_eta_abs_mean": 0.010635009035468102, "metrics/particle/matched/delta_phi_abs_mean": 0.011110801249742508, "metrics/particle/matched/delta_r_mean": 0.01719982549548149, "metrics/particle/matched/energy_relative_abs_mean": 0.08485051244497299, "metrics/particle/matched/pid_accuracy": 0.9638394713401794, "metrics/particle/matched/pt_relative_abs_mean": 0.07830636948347092, "metrics/particle/matching/duplicate_fraction": 0.03830809146165848, "metrics/particle/matching/efficiency": 0.8175053000450134, "metrics/particle/matching/f1": 0.8083201050758362, "metrics/particle/matching/purity": 0.7993389964103699, "metrics/particle/matching/target_coverage_dr0p05": 0.792098879814148, "metrics/particle/matching/target_coverage_dr0p10": 0.860185444355011, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7193344235420227, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8589155673980713, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02061733417212963, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08290664106607437, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07983822375535965, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022671379148960114, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13857710361480713, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031979153864085674, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3400401473045349, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.005437242798507214, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2299407720565796, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8443.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3080569803714752, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1404201984405518, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.287064790725708, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1686909198760986, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.14528211951255798, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.586974143981934, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015267899259924889, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8140223026275635, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.024266378954052925, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.891055107116699, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10545.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7854683995246887, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.5971317291259766, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.059572219848633, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6770124435424805, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.6002017259597778, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07446126639842987, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.86445951461792, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5923892855644226, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.965041160583496, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028026381507515907, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09839896112680435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09868639707565308, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017959771677851677, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1598127782344818, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020066889002919197, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0008851289749146, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.00555481156334281, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.099133014678955, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1794.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8322374820709229, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7198487520217896, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12158.9716796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12158.802734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2762034833431244, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 657805.5625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08501195162534714, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.165527582168579, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13087767362594604, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.253327369689941, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2929.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.583984375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.5500941276550293, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.5936279296875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.266655683517456, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02995729073882103, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.14993731677532196, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.414031028747559, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.13192179799079895, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.92696189880371, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014871968887746334, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08540308475494385, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07660526037216187, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.022736141458153725, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13512659072875977, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006312467157840729, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1168929785490036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.024076156318187714, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5716611742973328, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19010.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15303799510002136, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8653204441070557, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.717224657535553, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6231277585029602, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1000918298959732, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.343347549438477, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028466482181102037, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27619627118110657, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.014506455510854721, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.98870849609375, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21780.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8659014105796814, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.336268424987793, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.940805435180664, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.285662651062012, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.25604724884033203, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.9286863803863525, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.5680670738220215, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.11436834931373596, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.176712989807129, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01733742281794548, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08547957241535187, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07889299839735031, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02242446504533291, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13776226341724396, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025643655098974705, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2355346530675888, "metrics/particle/momentum/all/accepted/pz_bias": -0.013738701120018959, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1127697229385376, "metrics/particle/momentum/all/accepted/samples": 29247.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25583627820014954, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0186034440994263, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1011.0291137695312, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1010.9215698242188, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12824076414108276, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 189606.421875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013388551771640778, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6771217584609985, "metrics/particle/momentum/all/assigned/pz_bias": -0.012577362358570099, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.47236704826355, "metrics/particle/momentum/all/assigned/samples": 35254.0, "metrics/particle/momentum/all/efficiency": 0.8175262212753296, "metrics/particle/momentum/all/event/energy_bias": 9.48349380493164, "metrics/particle/momentum/all/event/p3_error_mean": 16.591495513916016, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.437983512878418, "metrics/particle/momentum/all/event/px_bias": 0.8262916803359985, "metrics/particle/momentum/all/event/py_bias": -2.004162549972534, "metrics/particle/momentum/all/event/pz_abs_mean": 12.281009674072266, "metrics/particle/momentum/all/event/pz_bias": -0.8386794328689575, "metrics/particle/momentum/all/event/pz_residual_rms": 22.49939727783203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008438016287982464, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03458685800433159, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022390762344002724, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0045566800981760025, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06383681297302246, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010471849236637354, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11030974984169006, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.020026471465826035, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7526435256004333, "metrics/particle/momentum/class_1/accepted/samples": 16234.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.048110757023096085, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.19275599718093872, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.19406968355178833, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.16540394723415375, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022343389689922333, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 9.055941581726074, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004741302691400051, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2577365040779114, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.003334199544042349, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.6332831382751465, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9621288180351257, "metrics/particle/momentum/class_1/event/energy_bias": 6.273377418518066, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.02698802947998, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.5771098136901855, "metrics/particle/momentum/class_1/event/px_bias": -0.6153287887573242, "metrics/particle/momentum/class_1/event/py_bias": -0.48275843262672424, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.363932609558105, "metrics/particle/momentum/class_1/event/pz_bias": -0.3304479122161865, "metrics/particle/momentum/class_1/event/pz_residual_rms": 19.668333053588867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04031025245785713, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19885170459747314, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19884489476680756, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03954783082008362, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2374553233385086, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031125299632549286, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.634656310081482, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.024763286113739014, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.456190347671509, "metrics/particle/momentum/class_2/accepted/samples": 1253.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9506711363792419, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7891232371330261, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12820.8193359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12820.3330078125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3817068636417389, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 675568.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08030248433351517, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.3245606422424316, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.18479005992412567, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.94268274307251, "metrics/particle/momentum/class_2/assigned/samples": 2777.0, "metrics/particle/momentum/class_2/efficiency": 0.4253224730491638, "metrics/particle/momentum/class_2/event/energy_bias": -2.4352352619171143, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.2073974609375, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.204572677612305, "metrics/particle/momentum/class_2/event/px_bias": 0.9042773246765137, "metrics/particle/momentum/class_2/event/py_bias": -1.386615514755249, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.903644561767578, "metrics/particle/momentum/class_2/event/pz_bias": -1.0200730562210083, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.199289321899414, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027946794405579567, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1467517912387848, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14871567487716675, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05588148534297943, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19182685017585754, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016891892300918698, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26064321398735046, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.002037370577454567, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.020216941833496, "metrics/particle/momentum/class_3/accepted/samples": 11248.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36693325638771057, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.988882303237915, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.3788022994995117, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.25060772895813, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20325365662574768, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 32.386680603027344, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011088977567851543, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6676263213157654, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.017003774642944336, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.928790330886841, "metrics/particle/momentum/class_3/assigned/samples": 15060.0, "metrics/particle/momentum/class_3/efficiency": 0.7298209071159363, "metrics/particle/momentum/class_3/event/energy_bias": 7.145792484283447, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.144020080566406, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.708401679992676, "metrics/particle/momentum/class_3/event/px_bias": 0.31297433376312256, "metrics/particle/momentum/class_3/event/py_bias": -0.4235140085220337, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.378785133361816, "metrics/particle/momentum/class_3/event/pz_bias": 0.30410051345825195, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.740388870239258, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011553196236491203, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10209164768457413, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.053858038038015366, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.008350622840225697, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10320866107940674, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2448526918888092, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.010057582519948483, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9220402836799622, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10371498018503189, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3979544639587402, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8224994540214539, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7690378427505493, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.656902313232422, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.36412689089775085, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.11867639422416687, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.201529622077942, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.935359001159668, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9606893062591553, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2693113088607788, "metrics/particle/momentum/class_4/event/px_bias": 0.1337248831987381, "metrics/particle/momentum/class_4/event/py_bias": 0.17137809097766876, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2478556632995605, "metrics/particle/momentum/class_4/event/pz_bias": 0.3180110454559326, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.646227836608887, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007706565782427788, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025921473279595375, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022453438490629196, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0004754046385642141, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06702909618616104, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2033347487449646, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1090005412697792, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5826140642166138, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02146126888692379, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04095961153507233, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.037506699562072754, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0052580335177481174, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12757518887519836, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5708236694335938, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.47856274247169495, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1760172843933105, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.5650813579559326, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1222258806228638, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7776615619659424, "metrics/particle/momentum/class_5/event/px_bias": 0.09064387530088425, "metrics/particle/momentum/class_5/event/py_bias": 0.1173473596572876, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6754940748214722, "metrics/particle/momentum/class_5/event/pz_bias": -0.11027001589536667, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4982340335845947, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.283219575881958, "step_valid_time": 3554.265832424164, "step_total_time": 3555.5490522384644, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0121341650958984, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08635108623551846, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3014, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2855, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9505804311774461, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9508958195089582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9507380991872616, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0491041804910418, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9469320066334992, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472461844724619, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9470890694974291, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9505804311774461, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0207829041101828, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10163140286442707, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2110, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9383044829116733, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9416481069042316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9399733214762116, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.058351893095768374, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9365290723479804, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9398663697104677, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.938194753223655, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9383044829116733, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0239661969675524, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10029028706876197, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1170, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.914396887159533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9165366614664586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9154655239579276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08346333853354135, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9105058365758755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9126365054602185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9115699259836385, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.914396887159533} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_2000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..467bbb28f55675d5b65a9da3e68db4865d0933f4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.09475080538184574, "train": {"Classification_binary": 3.965373992919922, "Classification": 0.03736860305070877, "Regression_pt": 0.2205420881509781, "Regression_eta": 0.0163961760699749, "Regression_sin_phi": 0.011190828867256641, "Regression_cos_phi": 0.011877315118908882, "Regression_energy": 0.22019444406032562, "Total": 13.64295768737793, "Total_unweighted": 4.482943448238075}, "train_group": {}, "valid": {"Classification_binary": 3.9638328552246094, "Classification": 0.038231549163659416, "Regression_pt": 0.22835477193196616, "Regression_eta": 0.015388675034046173, "Regression_sin_phi": 0.012310519814491272, "Regression_cos_phi": 0.012366215387980143, "Regression_energy": 0.23131360610326132, "Total": 14.00766372680664, "Total_unweighted": 4.501798192660014, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03949699550867081, "diagnostic/loss/clic_pf/Classification_binary": 3.973606586456299, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.012066423892974854, "diagnostic/loss/clic_pf/Regression_energy": 0.2333308309316635, "diagnostic/loss/clic_pf/Regression_eta": 0.014024048112332821, "diagnostic/loss/clic_pf/Regression_pt": 0.23144593834877014, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.012116122990846634, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05575172230601311, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.24744415283203125, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07130534201860428, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5544610619544983, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.028088826686143875, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2497403472661972, "diagnostic/regression/clic_pf/pt_residual_mean": 0.09693336486816406, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5609506964683533, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9560896158218384, "metrics/particle/class_1/pid_accuracy": 0.9852176904678345, "metrics/particle/class_2/efficiency": 0.20049616694450378, "metrics/particle/class_2/pid_accuracy": 0.2424072027206421, "metrics/particle/class_3/efficiency": 0.6440780162811279, "metrics/particle/class_3/pid_accuracy": 0.9812596440315247, "metrics/particle/class_4/efficiency": 0.9593023061752319, "metrics/particle/class_4/pid_accuracy": 0.06565656512975693, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.1454918086528778, "metrics/particle/count/bias_mean": 0.49934640526771545, "metrics/particle/count/mae": 4.469281196594238, "metrics/particle/count/prediction_mean": 70.49869537353516, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.10624093562364578, "metrics/particle/event/energy_response_mean": 1.0256400108337402, "metrics/particle/event/met_abs_error": 13.867768287658691, "metrics/particle/event/scalar_pt_relative_abs_error": 0.09204525500535965, "metrics/particle/event/scalar_pt_response_mean": 1.0305629968643188, "metrics/particle/event/vector_pt_closure": 0.1066495031118393, "metrics/particle/matched/delta_eta_abs_mean": 0.016598736867308617, "metrics/particle/matched/delta_phi_abs_mean": 0.015987427905201912, "metrics/particle/matched/delta_r_mean": 0.025744285434484482, "metrics/particle/matched/energy_relative_abs_mean": 0.088263601064682, "metrics/particle/matched/pid_accuracy": 0.9512531161308289, "metrics/particle/matched/pt_relative_abs_mean": 0.08529935032129288, "metrics/particle/matching/duplicate_fraction": 0.042832110077142715, "metrics/particle/matching/efficiency": 0.7585131525993347, "metrics/particle/matching/f1": 0.7558172941207886, "metrics/particle/matching/purity": 0.7531405687332153, "metrics/particle/matching/target_coverage_dr0p05": 0.6882230639457703, "metrics/particle/matching/target_coverage_dr0p10": 0.7984948754310608, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6667662858963013, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8096714019775391, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02792690321803093, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08899976313114166, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08831712603569031, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.028124172240495682, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1471773087978363, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00409673573449254, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.4072926938533783, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.013685022480785847, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4020740985870361, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7567.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4585626423358917, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.8565566539764404, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8862305879592896, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7391679286956787, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20065346360206604, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.38492774963379, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02700365148484707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2837915420532227, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.060270216315984726, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 6.259187698364258, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10406.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7039724588394165, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.556459426879883, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 15.369850158691406, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.94104528427124, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2807614803314209, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21828123927116394, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 13.179627418518066, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.914487361907959, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 21.80815887451172, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.035651735961437225, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09097565710544586, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09302162379026413, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025462185963988304, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15402619540691376, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020172910764813423, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0099120140075684, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.018253402784466743, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.266550302505493, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1388.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.0408179759979248, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.715040683746338, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 33256.32421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 33256.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3730919361114502, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1764728.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.1121760755777359, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.305854320526123, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.5704541802406311, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.154794692993164, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2817.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4518229067325592, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.8082200288772583, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.869064331054688, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.2348268032073975, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10983841866254807, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.2884260416030884, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 13.56860637664795, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.8890209197998047, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 31.143047332763672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02409130148589611, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08716102689504623, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08327067643404007, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0209676343947649, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1373046189546585, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000991571694612503, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13885515928268433, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0010947172995656729, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.679197371006012, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18153.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.22858282923698425, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.4323642253875732, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 1.0809072256088257, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.9490346312522888, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12425733357667923, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 15.225793838500977, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.007797994650900364, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4647931754589081, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04523231089115143, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.613778591156006, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21544.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8268652558326721, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.7533035278320312, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.972881317138672, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 15.123863220214844, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.7731845378875732, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7371230125427246, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.143909454345703, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -1.1851565837860107, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 21.353805541992188, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02575390599668026, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08786960691213608, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08517862111330032, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023195462301373482, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14101769030094147, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002840489847585559, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25838789343833923, "metrics/particle/momentum/all/accepted/pz_bias": 0.004021612461656332, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1849089860916138, "metrics/particle/momentum/all/accepted/samples": 27108.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.36322876811027527, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.7442820072174072, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2695.831787109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2695.689697265625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16728506982326508, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 502328.28125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.022003624588251114, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.02114737033844, "metrics/particle/momentum/all/assigned/pz_bias": -0.09228942543268204, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.284894943237305, "metrics/particle/momentum/all/assigned/samples": 34767.0, "metrics/particle/momentum/all/efficiency": 0.7577358484268188, "metrics/particle/momentum/all/event/energy_bias": 6.117982864379883, "metrics/particle/momentum/all/event/p3_error_mean": 35.68782043457031, "metrics/particle/momentum/all/event/pt_vector_error_mean": 19.040285110473633, "metrics/particle/momentum/all/event/px_bias": -2.1637845039367676, "metrics/particle/momentum/all/event/py_bias": 1.2438302040100098, "metrics/particle/momentum/all/event/pz_abs_mean": 26.026737213134766, "metrics/particle/momentum/all/event/pz_bias": -6.9886651039123535, "metrics/particle/momentum/all/event/pz_residual_rms": 42.4544677734375, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017664853483438492, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03827309235930443, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03400780260562897, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0210708174854517, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06988620012998581, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017997890245169401, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.16138067841529846, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.037140484899282455, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9765378832817078, "metrics/particle/momentum/class_1/accepted/samples": 16113.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0652836337685585, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1160472184419632, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15243802964687347, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12368563562631607, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0269661583006382, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.6699130535125732, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.006934155244380236, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.339003324508667, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.028140360489487648, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3653995990753174, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9549576044082642, "metrics/particle/momentum/class_1/event/energy_bias": 13.298454284667969, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.427127838134766, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 10.474372863769531, "metrics/particle/momentum/class_1/event/px_bias": -1.2791595458984375, "metrics/particle/momentum/class_1/event/py_bias": -0.7299530506134033, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.859195709228516, "metrics/particle/momentum/class_1/event/pz_bias": 2.827935218811035, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.895092010498047, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.047443583607673645, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.24523191154003143, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.23938001692295074, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08302492648363113, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2782101333141327, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.048821549862623215, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.124358654022217, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08724750578403473, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.444469451904297, "metrics/particle/momentum/class_2/accepted/samples": 594.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.5185515880584717, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.3124732971191406, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 36703.76953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 36703.01953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5783699154853821, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1854090.375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.13714733719825745, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 5.156400203704834, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.7128946781158447, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.97824478149414, "metrics/particle/momentum/class_2/assigned/samples": 2552.0, "metrics/particle/momentum/class_2/efficiency": 0.20162932574748993, "metrics/particle/momentum/class_2/event/energy_bias": -18.051631927490234, "metrics/particle/momentum/class_2/event/p3_error_mean": 28.913963317871094, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 14.241561889648438, "metrics/particle/momentum/class_2/event/px_bias": 0.15862524509429932, "metrics/particle/momentum/class_2/event/py_bias": 1.2392442226409912, "metrics/particle/momentum/class_2/event/pz_abs_mean": 21.80154037475586, "metrics/particle/momentum/class_2/event/pz_bias": -5.132691383361816, "metrics/particle/momentum/class_2/event/pz_residual_rms": 34.519981384277344, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0379372201859951, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.16158752143383026, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16143827140331268, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03409470617771149, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2038281112909317, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016195970820263028, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.29617032408714294, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.05229463055729866, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0144248008728027, "metrics/particle/momentum/class_3/accepted/samples": 9879.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5153915882110596, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.730560541152954, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.7354087829589844, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.5662131309509277, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2614542543888092, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 47.31031036376953, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01986755058169365, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.1029021739959717, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.1266881674528122, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 6.183877468109131, "metrics/particle/momentum/class_3/assigned/samples": 14798.0, "metrics/particle/momentum/class_3/efficiency": 0.6409940123558044, "metrics/particle/momentum/class_3/event/energy_bias": 18.108928680419922, "metrics/particle/momentum/class_3/event/p3_error_mean": 20.721145629882812, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 10.416332244873047, "metrics/particle/momentum/class_3/event/px_bias": -1.7578721046447754, "metrics/particle/momentum/class_3/event/py_bias": 0.7613967061042786, "metrics/particle/momentum/class_3/event/pz_abs_mean": 15.510370254516602, "metrics/particle/momentum/class_3/event/pz_bias": -4.584199905395508, "metrics/particle/momentum/class_3/event/pz_residual_rms": 25.494033813476562, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02159462310373783, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05376749485731125, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.055624738335609436, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0066695730201900005, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09791253507137299, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.005797101650387049, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.35146573185920715, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.12613607943058014, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1688642501831055, "metrics/particle/momentum/class_4/accepted/samples": 345.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.05920420214533806, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.3405979573726654, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3019532561302185, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.24732737243175507, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03038674034178257, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.865295886993408, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005524862091988325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.42015036940574646, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07560471445322037, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3272444009780884, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9530386924743652, "metrics/particle/momentum/class_4/event/energy_bias": -3.6842315196990967, "metrics/particle/momentum/class_4/event/p3_error_mean": 5.544122695922852, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 4.111763954162598, "metrics/particle/momentum/class_4/event/px_bias": 0.41683250665664673, "metrics/particle/momentum/class_4/event/py_bias": -0.0030249357223510742, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.987999200820923, "metrics/particle/momentum/class_4/event/pz_bias": 0.24111025035381317, "metrics/particle/momentum/class_4/event/pz_residual_rms": 9.311946868896484, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.017455806955695152, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026747601106762886, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02725130319595337, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.003035431494936347, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0609159916639328, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.5370668768882751, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.20056520402431488, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.3513489961624146, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.018154427409172058, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04298899322748184, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04428565874695778, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0005338031332939863, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12472927570343018, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.8255771398544312, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.0921361967921257, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7430295944213867, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -3.5535378456115723, "metrics/particle/momentum/class_5/event/p3_error_mean": 4.492635726928711, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 3.352220058441162, "metrics/particle/momentum/class_5/event/px_bias": 0.29778948426246643, "metrics/particle/momentum/class_5/event/py_bias": -0.02383265271782875, "metrics/particle/momentum/class_5/event/pz_abs_mean": 2.298501491546631, "metrics/particle/momentum/class_5/event/pz_bias": -0.3408191204071045, "metrics/particle/momentum/class_5/event/pz_residual_rms": 8.066916465759277, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2174928188323975, "step_valid_time": 464.0099880695343, "step_total_time": 465.22748160362244, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000541306029505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18699010214946188, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2742, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2581, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.909452736318408, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9117206982543641, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8560530679933664, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8603333333333333, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.858187863674148, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.909452736318408, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0110215107834661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20335026806104217, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2231, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1979, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1927, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8783843763870395, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8870461676378306, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8826940231935771, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11295383236216944, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8553040390590324, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8637382339757956, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.85950044603033, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8783843763870395, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0119011064126053, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2161566969893326, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1086, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1039, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.845136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8477751756440282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8464536243180046, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1522248243559719, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8085603112840467, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8110850897736144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8098207326578332, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.845136186770428} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_20000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..ea55b5c508d1272381360b0f79e256a15eae3071 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9475080538184575, "train": {"Classification_binary": 2.8667807579040527, "Classification": 0.024687986820936203, "Regression_pt": 0.13674843311309814, "Regression_eta": 0.010143701918423176, "Regression_sin_phi": 0.008480501361191273, "Regression_cos_phi": 0.00845685787498951, "Regression_energy": 0.13932517170906067, "Total": 9.137011528015137, "Total_unweighted": 3.1946234107017517}, "train_group": {}, "valid": {"Classification_binary": 2.782168388366699, "Classification": 0.022669727603594463, "Regression_pt": 0.14369367559750876, "Regression_eta": 0.011435961971680323, "Regression_sin_phi": 0.008912250399589539, "Regression_cos_phi": 0.009036778782804808, "Regression_energy": 0.14656909306844076, "Total": 9.30284563700358, "Total_unweighted": 3.124485875790318, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.023590829223394394, "diagnostic/loss/clic_pf/Classification_binary": 2.790567398071289, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008721524849534035, "diagnostic/loss/clic_pf/Regression_energy": 0.14955776929855347, "diagnostic/loss/clic_pf/Regression_eta": 0.010903050191700459, "diagnostic/loss/clic_pf/Regression_pt": 0.1467835158109665, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008639421314001083, "diagnostic/regression/clic_pf/energy_pred_mean": 0.044919583946466446, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19931338727474213, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06047321483492851, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4428001940250397, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.002487060148268938, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1960250735282898, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07133159786462784, "diagnostic/regression/clic_pf/pt_residual_rms": 0.44776666164398193, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9614367485046387, "metrics/particle/class_1/pid_accuracy": 0.9884632229804993, "metrics/particle/class_2/efficiency": 0.40200722217559814, "metrics/particle/class_2/pid_accuracy": 0.7068723440170288, "metrics/particle/class_3/efficiency": 0.7402091026306152, "metrics/particle/class_3/pid_accuracy": 0.9843049049377441, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.28347280621528625, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6069246530532837, "metrics/particle/count/bias_mean": -0.12614379823207855, "metrics/particle/count/mae": 3.4372549057006836, "metrics/particle/count/prediction_mean": 69.87319946289062, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05485490337014198, "metrics/particle/event/energy_response_mean": 0.9983518719673157, "metrics/particle/event/met_abs_error": 5.317289352416992, "metrics/particle/event/scalar_pt_relative_abs_error": 0.042856063693761826, "metrics/particle/event/scalar_pt_response_mean": 1.0037857294082642, "metrics/particle/event/vector_pt_closure": 0.04610900953412056, "metrics/particle/matched/delta_eta_abs_mean": 0.010022456757724285, "metrics/particle/matched/delta_phi_abs_mean": 0.011586896143853664, "metrics/particle/matched/delta_r_mean": 0.01711537316441536, "metrics/particle/matched/energy_relative_abs_mean": 0.08680527657270432, "metrics/particle/matched/pid_accuracy": 0.9655726552009583, "metrics/particle/matched/pt_relative_abs_mean": 0.07907719165086746, "metrics/particle/matching/duplicate_fraction": 0.02941836789250374, "metrics/particle/matching/efficiency": 0.8190646171569824, "metrics/particle/matching/f1": 0.8198032975196838, "metrics/particle/matching/purity": 0.8205432891845703, "metrics/particle/matching/target_coverage_dr0p05": 0.7854975461959839, "metrics/particle/matching/target_coverage_dr0p10": 0.853434681892395, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7233120799064636, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.859130322933197, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019734064117074013, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08624690771102905, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08109267055988312, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018037598580121994, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13718919456005096, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004271476063877344, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3613455891609192, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.018857473507523537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3044233322143555, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8428.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3119485080242157, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8817130327224731, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8940203785896301, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7632639408111572, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13508839905261993, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.145998001098633, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016813989728689194, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8142607808113098, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06900612264871597, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.699878454208374, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10408.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7840729355812073, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.37572041153907776, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.788296699523926, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.4982104301452637, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.4837666153907776, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.04265040159225464, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.648880958557129, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0540926456451416, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.52794075012207, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029028167948126793, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09813006967306137, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09639529138803482, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0190143920481205, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15361298620700836, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020749280229210854, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0495026111602783, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.017513440921902657, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.9241576194763184, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1735.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8018500208854675, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5404292345046997, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 26505.3359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 26505.16015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2735314965248108, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1408843.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08067940920591354, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.085205554962158, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.016861237585544586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.226036071777344, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2826.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5647786259651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.684117078781128, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.169553756713867, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.324789047241211, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07951068133115768, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.15534138679504395, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.0018949508667, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.33784037828445435, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.160598754882812, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014970461837947369, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08767195045948029, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07823754847049713, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.0177006796002388, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13430067896842957, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005228758091107011, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.117278553545475, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00859017577022314, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5956296920776367, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19125.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15202027559280396, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7105539441108704, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5038905143737793, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3986533284187317, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08961500972509384, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.972481727600098, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031427647918462753, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27056148648262024, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.016097743064165115, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.017939329147339, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21637.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.871139645576477, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.6919610500335693, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.549962043762207, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.205838203430176, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4834933876991272, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.05905282497406, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.7037503719329834, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.1313233375549316, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.362605094909668, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017174018546938896, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0878814160823822, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0801347941160202, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01787545531988144, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13635146617889404, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027997815050184727, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24273626506328583, "metrics/particle/momentum/all/accepted/pz_bias": 0.009998363442718983, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2780954837799072, "metrics/particle/momentum/all/accepted/samples": 29288.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.25241750478744507, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8288944363594055, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2148.613037109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2148.49462890625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11809239536523819, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 401066.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013506925664842129, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.660943329334259, "metrics/particle/momentum/all/assigned/pz_bias": 0.029218338429927826, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.4059605598449707, "metrics/particle/momentum/all/assigned/samples": 34871.0, "metrics/particle/momentum/all/efficiency": 0.818672239780426, "metrics/particle/momentum/all/event/energy_bias": -1.367876410484314, "metrics/particle/momentum/all/event/p3_error_mean": 15.963964462280273, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.8127617835998535, "metrics/particle/momentum/all/event/px_bias": -0.0797838494181633, "metrics/particle/momentum/all/event/py_bias": -1.2570445537567139, "metrics/particle/momentum/all/event/pz_abs_mean": 12.053318977355957, "metrics/particle/momentum/all/event/pz_bias": 2.523256301879883, "metrics/particle/momentum/all/event/pz_residual_rms": 21.99102783203125, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008500732481479645, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04074317216873169, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02685297094285488, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012032010592520237, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06800994277000427, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013582762330770493, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13239894807338715, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.011226753704249859, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8827431797981262, "metrics/particle/momentum/class_1/accepted/samples": 16197.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05115586891770363, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.21525707840919495, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2046772986650467, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.17654132843017578, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02418350987136364, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 10.003127098083496, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005275324452668428, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29010799527168274, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.044741712510585785, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.839411497116089, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9599359631538391, "metrics/particle/momentum/class_1/event/energy_bias": 5.961840629577637, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.770879745483398, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.4715962409973145, "metrics/particle/momentum/class_1/event/px_bias": -0.5639357566833496, "metrics/particle/momentum/class_1/event/py_bias": -0.560549259185791, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.120506286621094, "metrics/particle/momentum/class_1/event/pz_bias": 1.0682446956634521, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.735958099365234, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0402042455971241, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20955143868923187, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20960034430027008, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05213677138090134, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24801595509052277, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03474576398730278, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7044905424118042, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.03937828168272972, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.61220645904541, "metrics/particle/momentum/class_2/accepted/samples": 1180.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9935610890388489, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7902640700340271, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 28103.765625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 28103.212890625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.39737334847450256, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1450775.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08555346727371216, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.383028745651245, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04788192734122276, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.743684768676758, "metrics/particle/momentum/class_2/assigned/samples": 2665.0, "metrics/particle/momentum/class_2/efficiency": 0.40054312348365784, "metrics/particle/momentum/class_2/event/energy_bias": -11.144231796264648, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.05047607421875, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.622337818145752, "metrics/particle/momentum/class_2/event/px_bias": 0.10726367682218552, "metrics/particle/momentum/class_2/event/py_bias": -1.105675458908081, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.173383712768555, "metrics/particle/momentum/class_2/event/pz_bias": 0.6075260639190674, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.091028213500977, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02743048034608364, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1424066424369812, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1436777263879776, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03348957374691963, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18534883856773376, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016669591423124075, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2468673139810562, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.013396183028817177, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.3049832582473755, "metrics/particle/momentum/class_3/accepted/samples": 11398.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3543849587440491, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5367733240127563, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6261509656906128, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4801044464111328, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17760799825191498, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.076677322387695, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010276519693434238, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6010999083518982, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.014689038507640362, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.673626661300659, "metrics/particle/momentum/class_3/assigned/samples": 14791.0, "metrics/particle/momentum/class_3/efficiency": 0.7395535707473755, "metrics/particle/momentum/class_3/event/energy_bias": 5.425757884979248, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.422953605651855, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.556626319885254, "metrics/particle/momentum/class_3/event/px_bias": 0.06846268475055695, "metrics/particle/momentum/class_3/event/py_bias": 0.265000581741333, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.6629462242126465, "metrics/particle/momentum/class_3/event/pz_bias": 0.6393528580665588, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.535201072692871, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01174995955079794, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1146654263138771, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06833608448505402, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.01997331529855728, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11930885165929794, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.30268850922584534, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.021389303728938103, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9924393892288208, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13289481401443481, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.1840031147003174, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.6552003026008606, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6045924425125122, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.905905723571777, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4035058617591858, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04538581520318985, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.192495346069336, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.9421861171722412, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.045016288757324, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2973334789276123, "metrics/particle/momentum/class_4/event/px_bias": 0.16755595803260803, "metrics/particle/momentum/class_4/event/py_bias": -0.005106048658490181, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3304829597473145, "metrics/particle/momentum/class_4/event/pz_bias": 0.36052748560905457, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.798821926116943, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007196507416665554, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.029238974675536156, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023794475942850113, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010000030510127544, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.062088772654533386, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2156623899936676, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01326833851635456, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6901383996009827, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007382288109511137, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04229671508073807, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03694382309913635, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.014125078916549683, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1249869093298912, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5529364347457886, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35387852787971497, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.672180652618408, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.6690566539764404, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1226650476455688, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7486487627029419, "metrics/particle/momentum/class_5/event/px_bias": 0.14086955785751343, "metrics/particle/momentum/class_5/event/py_bias": 0.14928559958934784, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6944657564163208, "metrics/particle/momentum/class_5/event/pz_bias": -0.152394637465477, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.622352361679077, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3061752319335938, "step_valid_time": 3710.9447412490845, "step_total_time": 3712.250916957855, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024310229925768, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07039800195599488, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2873, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552238805970149, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504950495049505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528535980148883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04950495049504951, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9529021558872305, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9481848184818482, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9505376344086022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552238805970149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042568474019373, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0914883714025807, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2123, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9422991566799822, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473449352967426, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9448153093012907, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052655064703257475, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400798934753661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9451137884872824, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9425901201602136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9422991566799822, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042725332244078, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08801854878167648, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9166666666666666, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9163098481899572, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08333333333333333, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9073929961089494, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9080996884735203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9077462047489295, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9159533073929961} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_21000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..5ebcbb26143ae5de5c34c19f67a4f8d340068f40 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9948834565093804, "train": {"Classification_binary": 2.7745282649993896, "Classification": 0.021146852523088455, "Regression_pt": 0.1415243148803711, "Regression_eta": 0.010259591042995453, "Regression_sin_phi": 0.00983637198805809, "Regression_cos_phi": 0.008507169783115387, "Regression_energy": 0.14050430059432983, "Total": 9.125163078308105, "Total_unweighted": 3.106306865811348}, "train_group": {}, "valid": {"Classification_binary": 2.753856658935547, "Classification": 0.022271305322647095, "Regression_pt": 0.14531699816385904, "Regression_eta": 0.011527732014656067, "Regression_sin_phi": 0.008845300103227297, "Regression_cos_phi": 0.008985806256532669, "Regression_energy": 0.14821634689966837, "Total": 9.302695592244467, "Total_unweighted": 3.0990201476961374, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022806216031312943, "diagnostic/loss/clic_pf/Classification_binary": 2.7581849098205566, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008640408515930176, "diagnostic/loss/clic_pf/Regression_energy": 0.15123148262500763, "diagnostic/loss/clic_pf/Regression_eta": 0.01076561026275158, "diagnostic/loss/clic_pf/Regression_pt": 0.14885905385017395, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008487481623888016, "diagnostic/regression/clic_pf/energy_pred_mean": 0.059137992560863495, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20440421998500824, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07469160854816437, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4504587948322296, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.017159514129161835, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20205152034759521, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08600405603647232, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4560973048210144, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9601049423217773, "metrics/particle/class_1/pid_accuracy": 0.9854451417922974, "metrics/particle/class_2/efficiency": 0.41012629866600037, "metrics/particle/class_2/pid_accuracy": 0.7277976274490356, "metrics/particle/class_3/efficiency": 0.724259078502655, "metrics/particle/class_3/pid_accuracy": 0.9812511205673218, "metrics/particle/class_4/efficiency": 0.9205426573753357, "metrics/particle/class_4/pid_accuracy": 0.324210524559021, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.6795918345451355, "metrics/particle/count/bias_mean": 0.10457516461610794, "metrics/particle/count/mae": 3.356862783432007, "metrics/particle/count/prediction_mean": 70.10391998291016, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06102697178721428, "metrics/particle/event/energy_response_mean": 1.0312124490737915, "metrics/particle/event/met_abs_error": 5.203115940093994, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048870205879211426, "metrics/particle/event/scalar_pt_response_mean": 1.0290571451187134, "metrics/particle/event/vector_pt_closure": 0.047794945538043976, "metrics/particle/matched/delta_eta_abs_mean": 0.01207714807242155, "metrics/particle/matched/delta_phi_abs_mean": 0.011957257054746151, "metrics/particle/matched/delta_r_mean": 0.019029535353183746, "metrics/particle/matched/energy_relative_abs_mean": 0.08758306503295898, "metrics/particle/matched/pid_accuracy": 0.9641062617301941, "metrics/particle/matched/pt_relative_abs_mean": 0.08055165410041809, "metrics/particle/matching/duplicate_fraction": 0.033181365579366684, "metrics/particle/matching/efficiency": 0.8121364116668701, "metrics/particle/matching/f1": 0.8115302324295044, "metrics/particle/matching/purity": 0.8109249472618103, "metrics/particle/matching/target_coverage_dr0p05": 0.7644048929214478, "metrics/particle/matching/target_coverage_dr0p10": 0.8511470556259155, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7132559418678284, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8540695905685425, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02095640078186989, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08911087363958359, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0843975841999054, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024902809411287308, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14230573177337646, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031280077528208494, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35787129402160645, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05618549883365631, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.197075366973877, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8312.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3199346363544464, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1397638320922852, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.138514757156372, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0198146104812622, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1462055742740631, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.094843864440918, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01546589843928814, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8590171933174133, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08180514723062515, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.186956882476807, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10410.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7732812166213989, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.7779814004898071, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.936002254486084, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5818707942962646, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.20439031720161438, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1543826311826706, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.77707576751709, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0500560998916626, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.092629432678223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029640531167387962, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10039688646793365, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09674398601055145, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024920202791690826, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15539796650409698, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016129031777381897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9297577738761902, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05855802446603775, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.829986810684204, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1674.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8024663925170898, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7213523387908936, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 72610.8359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 72610.671875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29088377952575684, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3892488.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08176757395267487, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.5443403720855713, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08811303228139877, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 13.749719619750977, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2874.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.544921875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 2.0028903484344482, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.4525728225708, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.3866839408874512, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.15070953965187073, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.04909641295671463, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.239744186401367, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3426760137081146, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.533475875854492, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017373863607645035, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0860859751701355, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07770327478647232, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021194884553551674, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13368718326091766, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006842105067335069, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1166577935218811, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007865930907428265, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5764933228492737, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19000.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1532336324453354, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7655885219573975, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5556496381759644, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4568502604961395, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09337586164474487, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.109292984008789, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028659917879849672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.269147664308548, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.013928921893239021, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6915379762649536, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21633.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8654459118843079, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.709684133529663, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.49024772644043, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.263673782348633, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.9998552799224854, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.12092120945453644, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.495922565460205, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.158230721950531, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.593358039855957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019109610468149185, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08777987957000732, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08072256296873093, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02247331105172634, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13753847777843475, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002276961226016283, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2327861338853836, "metrics/particle/momentum/all/accepted/pz_bias": 0.014337519183754921, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0446515083312988, "metrics/particle/momentum/all/accepted/samples": 28986.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2563711702823639, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9558118581771851, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5977.24365234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5977.13330078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12538304924964905, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1116740.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01311681978404522, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7145881056785583, "metrics/particle/momentum/all/assigned/pz_bias": 0.008506756275892258, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.74976110458374, "metrics/particle/momentum/all/assigned/samples": 34917.0, "metrics/particle/momentum/all/efficiency": 0.8102306127548218, "metrics/particle/momentum/all/event/energy_bias": 6.490556240081787, "metrics/particle/momentum/all/event/p3_error_mean": 16.89296531677246, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.787710189819336, "metrics/particle/momentum/all/event/px_bias": -1.3549550771713257, "metrics/particle/momentum/all/event/py_bias": -0.32440024614334106, "metrics/particle/momentum/all/event/pz_abs_mean": 13.045384407043457, "metrics/particle/momentum/all/event/pz_bias": 1.2345014810562134, "metrics/particle/momentum/all/event/pz_residual_rms": 24.15238380432129, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00824439525604248, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03800509124994278, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.026602383702993393, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.012972179800271988, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0674917921423912, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012989422539249063, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12078163772821426, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0005030224565416574, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8089802265167236, "metrics/particle/momentum/class_1/accepted/samples": 16167.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.052430782467126846, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.14816802740097046, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15862509608268738, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1321898251771927, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.027921037748456, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.575257301330566, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004386744927614927, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2749631106853485, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005139967892318964, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.694152593612671, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.958158016204834, "metrics/particle/momentum/class_1/event/energy_bias": 7.119382858276367, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.562743186950684, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.681978225708008, "metrics/particle/momentum/class_1/event/px_bias": -0.25877654552459717, "metrics/particle/momentum/class_1/event/py_bias": -0.8158085942268372, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.781982421875, "metrics/particle/momentum/class_1/event/pz_bias": 0.10021466016769409, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.977739334106445, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043371379375457764, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1967555284500122, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1974167674779892, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03546637296676636, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23610307276248932, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028262676671147346, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6158326864242554, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.009047446772456169, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4262309074401855, "metrics/particle/momentum/class_2/accepted/samples": 1203.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9252853393554688, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8231638073921204, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 77835.703125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 77835.203125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37859007716178894, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4030159.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07870198041200638, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.381298303604126, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.011405074037611485, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.260483741760254, "metrics/particle/momentum/class_2/assigned/samples": 2681.0, "metrics/particle/momentum/class_2/efficiency": 0.4083503186702728, "metrics/particle/momentum/class_2/event/energy_bias": -6.155659198760986, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.289510726928711, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.336024284362793, "metrics/particle/momentum/class_2/event/px_bias": -0.8026013970375061, "metrics/particle/momentum/class_2/event/py_bias": -0.14121398329734802, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.58220386505127, "metrics/particle/momentum/class_2/event/pz_bias": -1.1685516834259033, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.92433738708496, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.032693177461624146, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14846812188625336, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14801040291786194, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.042512647807598114, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1901894062757492, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000990633969195187, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24421267211437225, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03771740570664406, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7700038552284241, "metrics/particle/momentum/class_3/accepted/samples": 11104.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3729310631752014, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8962033987045288, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8012884855270386, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6643531322479248, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19334816932678223, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.120264053344727, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011468663811683655, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.741655170917511, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.016571125015616417, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.05816125869751, "metrics/particle/momentum/class_3/assigned/samples": 14823.0, "metrics/particle/momentum/class_3/efficiency": 0.7204775214195251, "metrics/particle/momentum/class_3/event/energy_bias": 6.606805801391602, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.99384593963623, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.386722564697266, "metrics/particle/momentum/class_3/event/px_bias": -0.5216465592384338, "metrics/particle/momentum/class_3/event/py_bias": 0.5708657503128052, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.383585929870605, "metrics/particle/momentum/class_3/event/pz_bias": 2.1099700927734375, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.674670219421387, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012443769723176956, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1198166236281395, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.07385268062353134, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.029491476714611053, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1272091269493103, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.32083505392074585, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.015860090032219887, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9243274927139282, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.14726068079471588, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5257511138916016, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.38853120803833, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3398383855819702, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 16.05141258239746, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.40354037284851074, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.04362207278609276, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1665772199630737, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.7322317957878113, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0187156200408936, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2992284297943115, "metrics/particle/momentum/class_4/event/px_bias": 0.10982811450958252, "metrics/particle/momentum/class_4/event/py_bias": -0.03569479659199715, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3057781457901, "metrics/particle/momentum/class_4/event/pz_bias": 0.33506786823272705, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.749110221862793, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007118055131286383, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026142975315451622, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022903239354491234, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013739487156271935, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.056120507419109344, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18044286966323853, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0324285626411438, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5842976570129395, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.029126888141036034, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0439051054418087, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04055558890104294, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01358051598072052, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13545691967010498, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5935398936271667, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4487786293029785, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.2611541748046875, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3477417826652527, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1111968755722046, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7708345651626587, "metrics/particle/momentum/class_5/event/px_bias": 0.11824119091033936, "metrics/particle/momentum/class_5/event/py_bias": 0.09745137393474579, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6720718145370483, "metrics/particle/momentum/class_5/event/pz_bias": -0.14219939708709717, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.496415376663208, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.36763334274292, "step_valid_time": 3864.6784484386444, "step_total_time": 3866.046082019806, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011150029460314, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07296284139386655, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2856, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9505804311774461, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9464993394980185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9485354956147609, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.053500660501981503, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9472636815920398, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9431968295904888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.945225881184842, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9505804311774461, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0144467609796106, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09066149617103536, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2097, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9343098091433644, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9388938447814451, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9365962180200222, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.061106155218554864, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9307589880159787, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9353256021409456, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9330367074527253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9343098091433644, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0144145248543386, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09180861985761335, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1159, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9019455252918288, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9012441679626749, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9015947102294827, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09875583203732503, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8964980544747082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.895800933125972, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8961493582263711, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9019455252918288} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_22000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..ceecc59296a41b37d9230f37280e755135063da3 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0422588592003033, "train": {"Classification_binary": 2.7446296215057373, "Classification": 0.023689517751336098, "Regression_pt": 0.1614636778831482, "Regression_eta": 0.010616910643875599, "Regression_sin_phi": 0.009269943460822105, "Regression_cos_phi": 0.008323291316628456, "Regression_energy": 0.16259166598320007, "Total": 9.730694770812988, "Total_unweighted": 3.120584628544748}, "train_group": {}, "valid": {"Classification_binary": 2.716278076171875, "Classification": 0.022130923966566723, "Regression_pt": 0.14259034395217896, "Regression_eta": 0.011195593823989233, "Regression_sin_phi": 0.00856051097313563, "Regression_cos_phi": 0.008733540773391724, "Regression_energy": 0.14518725872039795, "Total": 9.129511515299479, "Total_unweighted": 3.054676248381535, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02269076555967331, "diagnostic/loss/clic_pf/Classification_binary": 2.722733497619629, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00840499997138977, "diagnostic/loss/clic_pf/Regression_energy": 0.14902114868164062, "diagnostic/loss/clic_pf/Regression_eta": 0.010460993275046349, "diagnostic/loss/clic_pf/Regression_pt": 0.1469600796699524, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008281461894512177, "diagnostic/regression/clic_pf/energy_pred_mean": 0.062271568924188614, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20014694333076477, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07782518863677979, "diagnostic/regression/clic_pf/energy_residual_rms": 0.45036759972572327, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.018820762634277344, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1979055106639862, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08766529709100723, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4559550881385803, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9608205556869507, "metrics/particle/class_1/pid_accuracy": 0.987359344959259, "metrics/particle/class_2/efficiency": 0.419936865568161, "metrics/particle/class_2/pid_accuracy": 0.7349624037742615, "metrics/particle/class_3/efficiency": 0.731134831905365, "metrics/particle/class_3/pid_accuracy": 0.9834615588188171, "metrics/particle/class_4/efficiency": 0.9127907156944275, "metrics/particle/class_4/pid_accuracy": 0.3322717547416687, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.686354398727417, "metrics/particle/count/bias_mean": -0.19542483985424042, "metrics/particle/count/mae": 3.2581698894500732, "metrics/particle/count/prediction_mean": 69.80392456054688, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05925505980849266, "metrics/particle/event/energy_response_mean": 1.0266915559768677, "metrics/particle/event/met_abs_error": 5.2974042892456055, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04669109731912613, "metrics/particle/event/scalar_pt_response_mean": 1.023637294769287, "metrics/particle/event/vector_pt_closure": 0.0459301620721817, "metrics/particle/matched/delta_eta_abs_mean": 0.010024712421000004, "metrics/particle/matched/delta_phi_abs_mean": 0.011676899157464504, "metrics/particle/matched/delta_r_mean": 0.017221124842762947, "metrics/particle/matched/energy_relative_abs_mean": 0.08453076332807541, "metrics/particle/matched/pid_accuracy": 0.9663440585136414, "metrics/particle/matched/pt_relative_abs_mean": 0.07764464616775513, "metrics/particle/matching/duplicate_fraction": 0.03158239647746086, "metrics/particle/matching/efficiency": 0.8161981105804443, "metrics/particle/matching/f1": 0.8173390030860901, "metrics/particle/matching/purity": 0.8184831738471985, "metrics/particle/matching/target_coverage_dr0p05": 0.7877197861671448, "metrics/particle/matching/target_coverage_dr0p10": 0.8543497323989868, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7208937406539917, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8561704754829407, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019656652584671974, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0836305171251297, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0786493569612503, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.024458110332489014, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13658487796783447, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031903579365462065, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.338583379983902, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0032119075767695904, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2290246486663818, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8463.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.30328619480133057, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1528160572052002, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.04939866065979, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9408693313598633, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1361371874809265, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.934548377990723, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016765663400292397, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.857914924621582, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.015583401545882225, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.0157294273376465, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10438.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7873290777206421, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.42634880542755127, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.923893928527832, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.05318021774292, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.03271865099668503, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5509193539619446, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.985040664672852, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7179470062255859, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.683855056762695, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027559656649827957, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10131260007619858, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09708351641893387, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.024745846167206764, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15767090022563934, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020348837599158287, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9415557384490967, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04004669189453125, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.974374771118164, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1720.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8149212598800659, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5357009172439575, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 72179.6015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 72179.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2727591097354889, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3857238.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08018207550048828, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.276789903640747, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07482834160327911, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.673678398132324, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2856.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5598958134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.1282269954681396, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.02672004699707, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2067097425460815, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.014921855181455612, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12189125269651413, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.869449138641357, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.4185888767242432, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.37071990966797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015196426771581173, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08446809649467468, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07634216547012329, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021975751966238022, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1334037333726883, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007362608448602259, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11056451499462128, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0008795743924565613, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5674660205841064, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19015.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1531200110912323, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.961491048336029, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7010219693183899, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6069874167442322, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09552404284477234, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.56881046295166, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00374151230789721, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27416208386421204, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0035018399357795715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8677910566329956, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21649.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8661291599273682, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.419282913208008, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.571216583251953, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.295315265655518, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.7786452174186707, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3776004910469055, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5703539848327637, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.24894869327545166, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.227364540100098, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01721751131117344, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08521760255098343, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07823273539543152, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02285844087600708, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1358734667301178, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026029180735349655, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22560760378837585, "metrics/particle/momentum/all/accepted/pz_bias": -0.0008552923100069165, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0810565948486328, "metrics/particle/momentum/all/accepted/samples": 29198.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2520679533481598, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0655746459960938, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5900.2109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5900.10693359375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1221417710185051, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1102746.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01387974712997675, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6939519047737122, "metrics/particle/momentum/all/assigned/pz_bias": -0.0007086149416863918, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.256366729736328, "metrics/particle/momentum/all/assigned/samples": 34943.0, "metrics/particle/momentum/all/efficiency": 0.8161565065383911, "metrics/particle/momentum/all/event/energy_bias": 5.973858833312988, "metrics/particle/momentum/all/event/p3_error_mean": 16.01237678527832, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.897733688354492, "metrics/particle/momentum/all/event/px_bias": -0.7964420914649963, "metrics/particle/momentum/all/event/py_bias": 0.2952100932598114, "metrics/particle/momentum/all/event/pz_abs_mean": 12.027341842651367, "metrics/particle/momentum/all/event/pz_bias": -0.45169323682785034, "metrics/particle/momentum/all/event/pz_residual_rms": 21.811248779296875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009174797683954239, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03622949868440628, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.024285610765218735, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0079946992918849, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06608612090349197, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001605731202289462, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1196986734867096, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005884265061467886, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9026512503623962, "metrics/particle/momentum/class_1/accepted/samples": 16192.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.055007074028253555, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.23650266230106354, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.19967301189899445, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1714857518672943, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02578236162662506, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 8.443588256835938, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005808440037071705, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.28003600239753723, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.029719598591327667, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7170536518096924, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9596396684646606, "metrics/particle/momentum/class_1/event/energy_bias": 4.972017288208008, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.406607627868652, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.390638828277588, "metrics/particle/momentum/class_1/event/px_bias": -1.0655839443206787, "metrics/particle/momentum/class_1/event/py_bias": -0.4232316315174103, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.722294330596924, "metrics/particle/momentum/class_1/event/pz_bias": 1.238779067993164, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.255035400390625, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04024697095155716, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.198524609208107, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19918379187583923, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03164597973227501, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23909786343574524, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03130016103386879, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5829881429672241, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.12232199311256409, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3267323970794678, "metrics/particle/momentum/class_2/accepted/samples": 1246.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9452404975891113, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8033278584480286, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 76831.65625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 76831.171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35929930210113525, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3979653.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08162504434585571, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4363746643066406, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1775721311569214, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.699861526489258, "metrics/particle/momentum/class_2/assigned/samples": 2683.0, "metrics/particle/momentum/class_2/efficiency": 0.42294636368751526, "metrics/particle/momentum/class_2/event/energy_bias": -5.155857086181641, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.16161823272705, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.6621479988098145, "metrics/particle/momentum/class_2/event/px_bias": 0.015546031296253204, "metrics/particle/momentum/class_2/event/py_bias": -0.4017269015312195, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.231800079345703, "metrics/particle/momentum/class_2/event/pz_bias": -1.0356159210205078, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.4427490234375, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026541979983448982, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1430656909942627, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14371639490127563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.050528187304735184, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18664710223674774, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000977343413978815, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22633787989616394, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0018597983289510012, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7745251059532166, "metrics/particle/momentum/class_3/accepted/samples": 11255.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.35663536190986633, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0246684551239014, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8521945476531982, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7279456853866577, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1910536289215088, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.850387573242188, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011182969436049461, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6771484017372131, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.001584777026437223, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.128570556640625, "metrics/particle/momentum/class_3/assigned/samples": 14844.0, "metrics/particle/momentum/class_3/efficiency": 0.7302750945091248, "metrics/particle/momentum/class_3/event/energy_bias": 7.2641682624816895, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.596261978149414, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.373371124267578, "metrics/particle/momentum/class_3/event/px_bias": 0.05445249378681183, "metrics/particle/momentum/class_3/event/py_bias": 0.9597628116607666, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.996142387390137, "metrics/particle/momentum/class_3/event/pz_bias": -0.8798119425773621, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.547477722167969, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012504630722105503, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12102114409208298, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06617528200149536, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.021796053275465965, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1139744222164154, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3003859221935272, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05262695625424385, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9468770623207092, "metrics/particle/momentum/class_4/accepted/samples": 326.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13452211022377014, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.840276002883911, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.397035598754883, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.3528897762298584, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.08287292718887329, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.550106048583984, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.44152772426605225, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.016447190195322037, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.289674997329712, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.90055251121521, "metrics/particle/momentum/class_4/event/energy_bias": -0.725590705871582, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0017056465148926, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3152968883514404, "metrics/particle/momentum/class_4/event/px_bias": 0.10927146673202515, "metrics/particle/momentum/class_4/event/py_bias": 0.10688531398773193, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2591564655303955, "metrics/particle/momentum/class_4/event/pz_bias": 0.35492873191833496, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.644002437591553, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0067282989621162415, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02534479834139347, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02079727128148079, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008945466950535774, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06547615677118301, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17524167895317078, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03310651704668999, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7346436977386475, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.006917001213878393, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03524811193346977, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030827846378087997, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.004030472598969936, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10592321306467056, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.509763777256012, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.36997148394584656, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.660905361175537, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.3808792233467102, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0424261093139648, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.709641695022583, "metrics/particle/momentum/class_5/event/px_bias": 0.08987196534872055, "metrics/particle/momentum/class_5/event/py_bias": 0.05352040007710457, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6482447385787964, "metrics/particle/momentum/class_5/event/pz_bias": -0.1299731433391571, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.485395908355713, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3361470699310303, "step_valid_time": 4005.8157415390015, "step_total_time": 4007.1518890857697, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009906027318112, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07165523476634728, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2875, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9535655058043118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9488448844884488, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9511993382961125, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05115511551155116, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9505804311774461, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9458745874587459, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.948221670802316, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9535655058043118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009519710849473, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08819280400541574, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2117, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2108, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.939636040834443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408888888888889, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.940262047523873, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.059111111111111114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.935641367066134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9368888888888889, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9362647124139463, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.939636040834443, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0111969652312636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09005811064467817, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1160, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9081712062256809, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9102964118564743, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9092325671990651, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08970358814352575, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9027237354085603, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9048361934477379, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9037787300350604, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9081712062256809} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_23000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..6dad2c486f4896e35c9cee28cebe67cd078baa40 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.089634261891226, "train": {"Classification_binary": 2.612760543823242, "Classification": 0.020136306062340736, "Regression_pt": 0.11733929067850113, "Regression_eta": 0.011111113242805004, "Regression_sin_phi": 0.00854923389852047, "Regression_cos_phi": 0.009342583827674389, "Regression_energy": 0.12194359302520752, "Total": 8.304754257202148, "Total_unweighted": 2.9011826645582914}, "train_group": {}, "valid": {"Classification_binary": 2.677776654561361, "Classification": 0.02175147831439972, "Regression_pt": 0.14112494389216104, "Regression_eta": 0.011448641618092855, "Regression_sin_phi": 0.008590089778105417, "Regression_cos_phi": 0.008722213407357534, "Regression_energy": 0.14418060580889383, "Total": 9.055473963419596, "Total_unweighted": 3.013594627380371, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022273730486631393, "diagnostic/loss/clic_pf/Classification_binary": 2.6873674392700195, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008467743173241615, "diagnostic/loss/clic_pf/Regression_energy": 0.1462714523077011, "diagnostic/loss/clic_pf/Regression_eta": 0.010546308942139149, "diagnostic/loss/clic_pf/Regression_pt": 0.14358237385749817, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008126424625515938, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04346393793821335, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19492179155349731, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05901756137609482, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4394722580909729, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0002651612740010023, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19190546870231628, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06857936829328537, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4445497691631317, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9613771438598633, "metrics/particle/class_1/pid_accuracy": 0.9860846996307373, "metrics/particle/class_2/efficiency": 0.421515554189682, "metrics/particle/class_2/pid_accuracy": 0.7239165306091309, "metrics/particle/class_3/efficiency": 0.7432482242584229, "metrics/particle/class_3/pid_accuracy": 0.9844850897789001, "metrics/particle/class_4/efficiency": 0.9156976938247681, "metrics/particle/class_4/pid_accuracy": 0.4296296238899231, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.6973415017127991, "metrics/particle/count/bias_mean": -0.5503268241882324, "metrics/particle/count/mae": 3.321568727493286, "metrics/particle/count/prediction_mean": 69.44902038574219, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05693821981549263, "metrics/particle/event/energy_response_mean": 1.0239638090133667, "metrics/particle/event/met_abs_error": 4.902008533477783, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04571887105703354, "metrics/particle/event/scalar_pt_response_mean": 1.0219457149505615, "metrics/particle/event/vector_pt_closure": 0.04675724729895592, "metrics/particle/matched/delta_eta_abs_mean": 0.009727119468152523, "metrics/particle/matched/delta_phi_abs_mean": 0.011021900922060013, "metrics/particle/matched/delta_r_mean": 0.016430549323558807, "metrics/particle/matched/energy_relative_abs_mean": 0.08406300097703934, "metrics/particle/matched/pid_accuracy": 0.9667457938194275, "metrics/particle/matched/pt_relative_abs_mean": 0.07655607163906097, "metrics/particle/matching/duplicate_fraction": 0.028929859399795532, "metrics/particle/matching/efficiency": 0.8218470811843872, "metrics/particle/matching/f1": 0.8250904679298401, "metrics/particle/matching/purity": 0.8283595442771912, "metrics/particle/matching/target_coverage_dr0p05": 0.7964966893196106, "metrics/particle/matching/target_coverage_dr0p10": 0.8552927374839783, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7271029353141785, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8600173592567444, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018869223073124886, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08208829164505005, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07649190723896027, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014514070935547352, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1330685019493103, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031832114327698946, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.337112158536911, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01767411269247532, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.264066457748413, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8482.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2948857247829437, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0624481439590454, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0063470602035522, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8872318863868713, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13097532093524933, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.674845695495605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01657671481370926, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8125407695770264, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08488749712705612, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.005605697631836, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10376.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7890966534614563, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.9689933061599731, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.391175270080566, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.345126152038574, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22453190386295319, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.08611112833023071, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.304041862487793, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.8995019197463989, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.550501823425293, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02646370604634285, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09605879336595535, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09297504276037216, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03158196061849594, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15062326192855835, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018824871629476547, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9216943383216858, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06113557890057564, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.875816822052002, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1753.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7439455986022949, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7282838821411133, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3197.09716796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3196.94970703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2750087082386017, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 171077.453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08016730844974518, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.367387294769287, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.26435771584510803, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.57242202758789, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2869.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5706380009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.3424758911132812, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.157428741455078, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.412519097328186, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.01287960447371006, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04063465818762779, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.947607040405273, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.359604835510254, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.59989356994629, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014482573606073856, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08423212915658951, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07516168802976608, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010865816846489906, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13176386058330536, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00041845382656902075, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1088809221982956, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0016672200290486217, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5215891003608704, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19118.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1516665369272232, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7304861545562744, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5365896821022034, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4300236701965332, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08789324760437012, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.668268203735352, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029189640190452337, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2689623534679413, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.008155045099556446, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.722035527229309, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21583.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8708208203315735, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7435262203216553, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.420478820800781, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.36003303527832, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4203145503997803, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.09760329872369766, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.0153355598449707, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.10395984351634979, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.78867244720459, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016465691849589348, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08431894332170486, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07660991698503494, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01315723080188036, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13334019482135773, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002316628582775593, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.223374143242836, "metrics/particle/momentum/all/accepted/pz_bias": 0.009844185784459114, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.064347267150879, "metrics/particle/momentum/all/accepted/samples": 29353.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24312430620193481, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9115795493125916, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 263.9972229003906, "metrics/particle/momentum/all/assigned/pt_relative_bias": 263.883544921875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11614218354225159, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 49101.38671875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013351326808333397, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6861424446105957, "metrics/particle/momentum/all/assigned/pz_bias": 0.042012881487607956, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.189255237579346, "metrics/particle/momentum/all/assigned/samples": 34828.0, "metrics/particle/momentum/all/efficiency": 0.8204891681671143, "metrics/particle/momentum/all/event/energy_bias": 4.054995536804199, "metrics/particle/momentum/all/event/p3_error_mean": 15.143921852111816, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.123071193695068, "metrics/particle/momentum/all/event/px_bias": -0.20866228640079498, "metrics/particle/momentum/all/event/py_bias": 0.22434909641742706, "metrics/particle/momentum/all/event/pz_abs_mean": 11.688051223754883, "metrics/particle/momentum/all/event/pz_bias": 2.155147075653076, "metrics/particle/momentum/all/event/pz_residual_rms": 20.076711654663086, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00833134911954403, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.036335289478302, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02294778637588024, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007617423310875893, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06377904117107391, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014205422485247254, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1211545467376709, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.011346990242600441, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8398329615592957, "metrics/particle/momentum/class_1/accepted/samples": 16191.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0537545308470726, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.14028292894363403, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1188313290476799, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09108738601207733, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.025490544736385345, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9801440238952637, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005216669756919146, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.25968626141548157, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01794343814253807, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.180933713912964, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.9595804214477539, "metrics/particle/momentum/class_1/event/energy_bias": 6.103332996368408, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.395305633544922, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.168933868408203, "metrics/particle/momentum/class_1/event/px_bias": -0.425760418176651, "metrics/particle/momentum/class_1/event/py_bias": 0.13077136874198914, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.004575729370117, "metrics/particle/momentum/class_1/event/pz_bias": 0.78038489818573, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.050779342651367, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0383295863866806, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20322416722774506, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20296812057495117, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04185568913817406, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2425583451986313, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.027821939438581467, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5510218143463135, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.030344102531671524, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3564248085021973, "metrics/particle/momentum/class_2/accepted/samples": 1258.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9186633825302124, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8165130019187927, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3417.583251953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3417.079345703125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3758389353752136, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 176941.078125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07904548943042755, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.319242000579834, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.28752100467681885, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.228331565856934, "metrics/particle/momentum/class_2/assigned/samples": 2682.0, "metrics/particle/momentum/class_2/efficiency": 0.4270196855068207, "metrics/particle/momentum/class_2/event/energy_bias": -7.5456743240356445, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.534341812133789, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.102902412414551, "metrics/particle/momentum/class_2/event/px_bias": 0.4205048382282257, "metrics/particle/momentum/class_2/event/py_bias": 0.16932743787765503, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.988541603088379, "metrics/particle/momentum/class_2/event/pz_bias": 1.2397152185440063, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.94850730895996, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025887610390782356, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13907980918884277, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14020061492919922, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.026926567777991295, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18197450041770935, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008775008609518409, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2182765156030655, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0027221182826906443, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7906718850135803, "metrics/particle/momentum/class_3/accepted/samples": 11396.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3424481451511383, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7910983562469482, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7523037195205688, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6087121963500977, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17525283992290497, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.89982032775879, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011063599027693272, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7004216909408569, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.028807571157813072, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.840817451477051, "metrics/particle/momentum/class_3/assigned/samples": 14733.0, "metrics/particle/momentum/class_3/efficiency": 0.7394238114356995, "metrics/particle/momentum/class_3/event/energy_bias": 6.007737636566162, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.319859504699707, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.188164710998535, "metrics/particle/momentum/class_3/event/px_bias": -0.3967224955558777, "metrics/particle/momentum/class_3/event/py_bias": -0.18275287747383118, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.826615333557129, "metrics/particle/momentum/class_3/event/pz_bias": -0.06721287965774536, "metrics/particle/momentum/class_3/event/pz_residual_rms": 13.283279418945312, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012087678536772728, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1257375180721283, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06181921064853668, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.021886153146624565, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10999715328216553, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3514677882194519, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.10593133419752121, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0927016735076904, "metrics/particle/momentum/class_4/accepted/samples": 329.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.12818117439746857, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.200648546218872, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.961637020111084, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9100048542022705, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.398929595947266, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.48824983835220337, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.11342088878154755, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3983266353607178, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.908839762210846, "metrics/particle/momentum/class_4/event/energy_bias": -0.2191356122493744, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9192208051681519, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2169344425201416, "metrics/particle/momentum/class_4/event/px_bias": 0.06229426711797714, "metrics/particle/momentum/class_4/event/py_bias": 0.013704560697078705, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.271978735923767, "metrics/particle/momentum/class_4/event/pz_bias": 0.32341089844703674, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.783192157745361, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0067816744558513165, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026436224579811096, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021143585443496704, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008207934908568859, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.060439031571149826, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.22787702083587646, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.006657725665718317, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8938040733337402, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02858801931142807, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04001479968428612, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03472492843866348, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0054845805279910564, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12730178236961365, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6486630439758301, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41799429059028625, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.353697299957275, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.29126548767089844, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1127433776855469, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7568061351776123, "metrics/particle/momentum/class_5/event/px_bias": 0.13102154433727264, "metrics/particle/momentum/class_5/event/py_bias": 0.09329861402511597, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6841404438018799, "metrics/particle/momentum/class_5/event/pz_bias": -0.12115111947059631, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5335299968719482, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.331824779510498, "step_valid_time": 4145.818786621094, "step_total_time": 4147.150611639023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033706614238054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0676037805323545, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2857, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2845, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9475953565505805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9475953565505805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9475953565505805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05240464344941957, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9436152570480929, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9436152570480929, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9436152570480929, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9475953565505805, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0082265398485117, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0842872678023957, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2117, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.939636040834443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9425645592163847, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9410980217826184, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.057435440783615316, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9383044829116733, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9412288512911843, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9397643920871305, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.939636040834443, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008771559669716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08790253658738734, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1161, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.909727626459144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9111457521434139, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.910436137071651, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08885424785658613, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9035019455252918, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9049103663289166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9042056074766355, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.909727626459144} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_24000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..7c3e7a09d5fc99536ea761daebfb519d7d56f9d1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.137009664582149, "train": {"Classification_binary": 2.6680421829223633, "Classification": 0.022089017555117607, "Regression_pt": 0.16217035055160522, "Regression_eta": 0.011261065490543842, "Regression_sin_phi": 0.009534155949950218, "Regression_cos_phi": 0.011544856242835522, "Regression_energy": 0.16260074079036713, "Total": 9.83336353302002, "Total_unweighted": 3.047242369502783}, "train_group": {}, "valid": {"Classification_binary": 2.6907946268717446, "Classification": 0.021839136878649395, "Regression_pt": 0.14093643426895142, "Regression_eta": 0.011163320392370224, "Regression_sin_phi": 0.008547294264038404, "Regression_cos_phi": 0.008713099484642347, "Regression_energy": 0.14397315184275308, "Total": 9.050076802571615, "Total_unweighted": 3.0259670640031495, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022765064612030983, "diagnostic/loss/clic_pf/Classification_binary": 2.71075177192688, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008247962221503258, "diagnostic/loss/clic_pf/Regression_energy": 0.14743351936340332, "diagnostic/loss/clic_pf/Regression_eta": 0.010481033474206924, "diagnostic/loss/clic_pf/Regression_pt": 0.1443808674812317, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008199082687497139, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03326410800218582, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19213838875293732, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04881773144006729, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4275408983230591, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.007398978341370821, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19069387018680573, "diagnostic/regression/clic_pf/pt_residual_mean": 0.061445556581020355, "diagnostic/regression/clic_pf/pt_residual_rms": 0.433439701795578, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9648955464363098, "metrics/particle/class_1/pid_accuracy": 0.9859294295310974, "metrics/particle/class_2/efficiency": 0.408660352230072, "metrics/particle/class_2/pid_accuracy": 0.7193708419799805, "metrics/particle/class_3/efficiency": 0.737924337387085, "metrics/particle/class_3/pid_accuracy": 0.983467698097229, "metrics/particle/class_4/efficiency": 0.9166666865348816, "metrics/particle/class_4/pid_accuracy": 0.353065550327301, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.6565040946006775, "metrics/particle/count/bias_mean": 0.24967320263385773, "metrics/particle/count/mae": 3.288888931274414, "metrics/particle/count/prediction_mean": 70.24901580810547, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.051421090960502625, "metrics/particle/event/energy_response_mean": 0.9889764189720154, "metrics/particle/event/met_abs_error": 4.784075736999512, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03990284353494644, "metrics/particle/event/scalar_pt_response_mean": 0.9988834261894226, "metrics/particle/event/vector_pt_closure": 0.04192255064845085, "metrics/particle/matched/delta_eta_abs_mean": 0.010829065926373005, "metrics/particle/matched/delta_phi_abs_mean": 0.01078770775347948, "metrics/particle/matched/delta_r_mean": 0.017152519896626472, "metrics/particle/matched/energy_relative_abs_mean": 0.08266234397888184, "metrics/particle/matched/pid_accuracy": 0.9653115272521973, "metrics/particle/matched/pt_relative_abs_mean": 0.0765988901257515, "metrics/particle/matching/duplicate_fraction": 0.030126255005598068, "metrics/particle/matching/efficiency": 0.8201664090156555, "metrics/particle/matching/f1": 0.8187063336372375, "metrics/particle/matching/purity": 0.817251443862915, "metrics/particle/matching/target_coverage_dr0p05": 0.7867767214775085, "metrics/particle/matching/target_coverage_dr0p10": 0.8560584187507629, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7265240550041199, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8613712787628174, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019910961389541626, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08088607341051102, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07749606668949127, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005868720356374979, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13463793694972992, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003181711072102189, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.34011638164520264, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.022960413247346878, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2505537271499634, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8486.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.297355592250824, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7346791625022888, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8364769220352173, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7046276926994324, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.132976695895195, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.42160415649414, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013660680502653122, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7368379831314087, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0037347255274653435, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5180437564849854, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10468.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7894687652587891, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.0395779609680176, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.74047327041626, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.0038630962371826, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09732861071825027, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.32896265387535095, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.735886096954346, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7157766819000244, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.599483489990234, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02869335375726223, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09054553508758545, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08907649666070938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0045171775855124, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1449110507965088, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022624434903264046, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9641522169113159, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0091129494830966, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.104573965072632, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1768.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.753516435623169, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0212428569793701, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6939.56689453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6939.38134765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2648386061191559, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 372376.84375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07150295376777649, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6993095874786377, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.24776607751846313, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.164144515991211, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2881.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5755208134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.474972248077393, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.273748874664307, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.095842957496643, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03873042017221451, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.13089680671691895, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.1438140869140625, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6253235340118408, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.291000366210938, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014863432385027409, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0825556218624115, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07488363236188889, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016539184376597404, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13185489177703857, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00036849864409305155, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10951732844114304, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009390560910105705, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5289391875267029, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18996.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15105094015598297, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5247530341148376, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.41983041167259216, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.31657692790031433, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09292872995138168, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.112943649291992, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0020784258376806974, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24048085510730743, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.017740845680236816, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5463643074035645, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21651.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8652637600898743, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8528870344161987, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.7501630783081055, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.570023536682129, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4836462140083313, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5605058670043945, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.216411828994751, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.0654359683394432, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.546607971191406, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017163759097456932, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08255419135093689, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07649943232536316, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012170733883976936, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13348893821239471, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002529914490878582, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22807657718658447, "metrics/particle/momentum/all/accepted/pz_bias": -0.013310661539435387, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1036291122436523, "metrics/particle/momentum/all/accepted/samples": 29250.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24440008401870728, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6284071207046509, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 571.7354125976562, "metrics/particle/momentum/all/assigned/pt_relative_bias": 571.6168823242188, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11905714124441147, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 106836.6484375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011257142759859562, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.591330885887146, "metrics/particle/momentum/all/assigned/pz_bias": -0.03248617798089981, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9769287109375, "metrics/particle/momentum/all/assigned/samples": 35000.0, "metrics/particle/momentum/all/efficiency": 0.8176100850105286, "metrics/particle/momentum/all/event/energy_bias": -5.661663055419922, "metrics/particle/momentum/all/event/p3_error_mean": 14.304360389709473, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.004549026489258, "metrics/particle/momentum/all/event/px_bias": -0.42504802346229553, "metrics/particle/momentum/all/event/py_bias": 1.0203652381896973, "metrics/particle/momentum/all/event/pz_abs_mean": 10.669657707214355, "metrics/particle/momentum/all/event/pz_bias": -2.2756640911102295, "metrics/particle/momentum/all/event/pz_residual_rms": 19.134973526000977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007457389030605555, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03275347501039505, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021864907816052437, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0006500103045254946, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06233062967658043, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011711044935509562, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10743071138858795, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0039188433438539505, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7881845831871033, "metrics/particle/momentum/class_1/accepted/samples": 16224.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.049882955849170685, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12074866145849228, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.13035184144973755, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09327343851327896, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022642403841018677, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.597102165222168, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047418647445738316, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24801714718341827, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0009937577415257692, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4473581314086914, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.961536169052124, "metrics/particle/momentum/class_1/event/energy_bias": 1.9894599914550781, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.654876708984375, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.017905235290527, "metrics/particle/momentum/class_1/event/px_bias": -0.6579310894012451, "metrics/particle/momentum/class_1/event/py_bias": -0.04169020056724548, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.191855430603027, "metrics/particle/momentum/class_1/event/pz_bias": -0.19500133395195007, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.9157075881958, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.041502177715301514, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1991448849439621, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19871638715267181, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03880841284990311, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.239020437002182, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029580935835838318, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.655019998550415, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05176916718482971, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5015370845794678, "metrics/particle/momentum/class_2/accepted/samples": 1217.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9641354084014893, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.679217517375946, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 7409.0458984375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 7408.5205078125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37991103529930115, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 384798.4375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07449962943792343, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.030360460281372, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.05740174278616905, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.886113166809082, "metrics/particle/momentum/class_2/assigned/samples": 2698.0, "metrics/particle/momentum/class_2/efficiency": 0.41310250759124756, "metrics/particle/momentum/class_2/event/energy_bias": -11.35683536529541, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.098983764648438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.021097660064697, "metrics/particle/momentum/class_2/event/px_bias": 0.07589870691299438, "metrics/particle/momentum/class_2/event/py_bias": -0.0898217111825943, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.543230056762695, "metrics/particle/momentum/class_2/event/pz_bias": -0.6912919282913208, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.23666000366211, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.028807539492845535, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14161573350429535, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14312471449375153, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03444010764360428, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18430061638355255, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0016815647250041366, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24606771767139435, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.03313782066106796, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9462981224060059, "metrics/particle/momentum/class_3/accepted/samples": 11299.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.340484082698822, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2037733793258667, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2616195678710938, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1219149827957153, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1836501657962799, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.677054405212402, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007456169929355383, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5441657304763794, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.06162681803107262, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2888686656951904, "metrics/particle/momentum/class_3/assigned/samples": 14887.0, "metrics/particle/momentum/class_3/efficiency": 0.7331300377845764, "metrics/particle/momentum/class_3/event/energy_bias": 4.883743762969971, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.212930679321289, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.450875759124756, "metrics/particle/momentum/class_3/event/px_bias": -0.05424324423074722, "metrics/particle/momentum/class_3/event/py_bias": 1.0665018558502197, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.466744899749756, "metrics/particle/momentum/class_3/event/pz_bias": -1.696131706237793, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.857089042663574, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011863467283546925, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10805226117372513, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.058717384934425354, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.009476515464484692, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1055072769522667, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2579198479652405, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.03181348741054535, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8685828447341919, "metrics/particle/momentum/class_4/accepted/samples": 330.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11385931074619293, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5447443127632141, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4077826142311096, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.34513264894485474, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.2095677852630615, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38324952125549316, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02082362025976181, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3348298072814941, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9116021990776062, "metrics/particle/momentum/class_4/event/energy_bias": -0.6205244660377502, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9723745584487915, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2710027694702148, "metrics/particle/momentum/class_4/event/px_bias": 0.10689768940210342, "metrics/particle/momentum/class_4/event/py_bias": 0.0028556305915117264, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2793289422988892, "metrics/particle/momentum/class_4/event/pz_bias": 0.4545022249221802, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.624220848083496, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006288084667176008, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028806624934077263, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02495267055928707, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.002288832562044263, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08134012669324875, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.27051064372062683, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1443740576505661, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.1999053955078125, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.006517650093883276, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03735039755702019, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03361198678612709, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.001522478531114757, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11771156638860703, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5309894680976868, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.40623900294303894, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7443721294403076, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.5575070381164551, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1097640991210938, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7692307233810425, "metrics/particle/momentum/class_5/event/px_bias": 0.10432995855808258, "metrics/particle/momentum/class_5/event/py_bias": 0.08251964300870895, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.665023684501648, "metrics/particle/momentum/class_5/event/pz_bias": -0.14774148166179657, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5146689414978027, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2618916034698486, "step_valid_time": 4286.126752614975, "step_total_time": 4287.388644695282, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998193824196054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06629728776479493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2867, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9509121061359868, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487094639311714, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498095080337916, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05129053606882859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9482587064676616, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9460622104566512, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.947159185025675, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9509121061359868, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968084134472106, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07758660116423499, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2115, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2113, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9387483355525965, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9454626732230667, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9420935412026726, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05453732677693339, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378606302707501, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9445686186857398, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9412026726057906, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9387483355525965, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985547005778903, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08382946388769408, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1269, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.914396887159533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9259259259259259, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9201252936570086, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07407407407407407, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.909727626459144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9211977935382191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9154267815191856, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.914396887159533} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_25000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..836c1d331522c724edaefaec27002dde6b09baa6 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1843850672730718, "train": {"Classification_binary": 2.6412672996520996, "Classification": 0.020335931330919266, "Regression_pt": 0.1264357715845108, "Regression_eta": 0.010225667618215084, "Regression_sin_phi": 0.008774121291935444, "Regression_cos_phi": 0.009542127139866352, "Regression_energy": 0.12827494740486145, "Total": 8.549853324890137, "Total_unweighted": 2.944855866022408}, "train_group": {}, "valid": {"Classification_binary": 2.6528008778889975, "Classification": 0.021483237544695537, "Regression_pt": 0.1381815473238627, "Regression_eta": 0.011245468010505041, "Regression_sin_phi": 0.008352174734075865, "Regression_cos_phi": 0.008587919175624847, "Regression_energy": 0.14098034302393594, "Total": 8.900607426961264, "Total_unweighted": 2.9816315677016973, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022153455764055252, "diagnostic/loss/clic_pf/Classification_binary": 2.6636157035827637, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008255857042968273, "diagnostic/loss/clic_pf/Regression_energy": 0.143866628408432, "diagnostic/loss/clic_pf/Regression_eta": 0.010628076270222664, "diagnostic/loss/clic_pf/Regression_pt": 0.141336128115654, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008000370115041733, "diagnostic/regression/clic_pf/energy_pred_mean": 0.043757542967796326, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19271451234817505, "diagnostic/regression/clic_pf/energy_residual_mean": 0.059311166405677795, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4380929470062256, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.001057379529811442, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19074445962905884, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06990192085504532, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4420516788959503, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9630071520805359, "metrics/particle/class_1/pid_accuracy": 0.9863146543502808, "metrics/particle/class_2/efficiency": 0.41576454043388367, "metrics/particle/class_2/pid_accuracy": 0.7360998392105103, "metrics/particle/class_3/efficiency": 0.7408772706985474, "metrics/particle/class_3/pid_accuracy": 0.9834172129631042, "metrics/particle/class_4/efficiency": 0.9215116500854492, "metrics/particle/class_4/pid_accuracy": 0.37223973870277405, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.676171064376831, "metrics/particle/count/bias_mean": -0.7699346542358398, "metrics/particle/count/mae": 3.386928081512451, "metrics/particle/count/prediction_mean": 69.22940826416016, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.052636753767728806, "metrics/particle/event/energy_response_mean": 0.9932081699371338, "metrics/particle/event/met_abs_error": 4.631197929382324, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04103679955005646, "metrics/particle/event/scalar_pt_response_mean": 1.0014151334762573, "metrics/particle/event/vector_pt_closure": 0.043392229825258255, "metrics/particle/matched/delta_eta_abs_mean": 0.011801929213106632, "metrics/particle/matched/delta_phi_abs_mean": 0.0110480897128582, "metrics/particle/matched/delta_r_mean": 0.018054811283946037, "metrics/particle/matched/energy_relative_abs_mean": 0.0819399505853653, "metrics/particle/matched/pid_accuracy": 0.9663209915161133, "metrics/particle/matched/pt_relative_abs_mean": 0.07646139711141586, "metrics/particle/matching/duplicate_fraction": 0.02756771631538868, "metrics/particle/matching/efficiency": 0.8211841583251953, "metrics/particle/matching/f1": 0.825725257396698, "metrics/particle/matching/purity": 0.830316960811615, "metrics/particle/matching/target_coverage_dr0p05": 0.7897086143493652, "metrics/particle/matching/target_coverage_dr0p10": 0.8560023903846741, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7271309494972229, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8588782548904419, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018826888874173164, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08160916715860367, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07734870910644531, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014889601618051529, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13478563725948334, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003762050298973918, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3416224420070648, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.013607366941869259, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2650558948516846, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8506.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2938687205314636, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.833028256893158, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8666735887527466, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7426990866661072, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12905718386173248, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.05830192565918, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013523956760764122, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7331117987632751, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.025057032704353333, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.500533103942871, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10352.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7913294434547424, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9402039051055908, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.691196441650391, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.020711898803711, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.23970754444599152, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.04322541877627373, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.7466840744018555, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3190447688102722, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.18610668182373, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026369286701083183, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09354285150766373, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09263332188129425, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013279876671731472, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15130838751792908, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022014308720827103, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9664498567581177, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.049837514758110046, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.443809747695923, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1817.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7322432398796082, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5388998985290527, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 62918.29296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 62918.12890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24885442852973938, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3351062.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07472682744264603, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8362531661987305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.03431854769587517, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.738370895385742, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2837.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5914713740348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.848144054412842, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.566156387329102, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1468843221664429, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.009062211960554123, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10198751091957092, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.430483818054199, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6510122418403625, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.072593688964844, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016886910423636436, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0811077132821083, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07445918023586273, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015483103692531586, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13102993369102478, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006850758800283074, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11617574840784073, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.041437260806560516, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5394962430000305, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18976.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1518511027097702, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6639361381530762, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5122218728065491, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.40974241495132446, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0902031883597374, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.12887191772461, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002650299808010459, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2637782096862793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04198305681347847, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6999783515930176, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21507.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8643527626991272, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.2278042733669281, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.62960147857666, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.199344158172607, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.565147876739502, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2784632742404938, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.530233383178711, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.8433864116668701, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.504858493804932, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018038174137473106, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0820244699716568, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07642515003681183, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01517416536808014, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13346810638904572, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0029011229053139687, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23435716331005096, "metrics/particle/momentum/all/accepted/pz_bias": -0.02769729122519493, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1783961057662964, "metrics/particle/momentum/all/accepted/samples": 29299.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2416810393333435, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7859303951263428, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5145.23876953125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5145.12451171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11476827412843704, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 958236.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01178810279816389, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6141543984413147, "metrics/particle/momentum/all/assigned/pz_bias": -0.0213540717959404, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.378079414367676, "metrics/particle/momentum/all/assigned/samples": 34696.0, "metrics/particle/momentum/all/efficiency": 0.8189797401428223, "metrics/particle/momentum/all/event/energy_bias": -4.0161519050598145, "metrics/particle/momentum/all/event/p3_error_mean": 14.513176918029785, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.599939346313477, "metrics/particle/momentum/all/event/px_bias": -0.8139176368713379, "metrics/particle/momentum/all/event/py_bias": -0.13325035572052002, "metrics/particle/momentum/all/event/pz_abs_mean": 11.3976411819458, "metrics/particle/momentum/all/event/pz_bias": -1.8134434223175049, "metrics/particle/momentum/all/event/pz_residual_rms": 19.795930862426758, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00861774105578661, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03174418583512306, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022501299157738686, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00037668782169930637, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06254081428050995, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016050373669713736, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1189763993024826, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.016536103561520576, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8361464738845825, "metrics/particle/momentum/class_1/accepted/samples": 16199.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05468769744038582, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.14319317042827606, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14553581178188324, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10959001630544662, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.024248532950878143, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.668846607208252, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0050987135618925095, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2509797215461731, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.023590439930558205, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0917303562164307, "metrics/particle/momentum/class_1/assigned/samples": 16867.0, "metrics/particle/momentum/class_1/efficiency": 0.9600545167922974, "metrics/particle/momentum/class_1/event/energy_bias": 0.8611090183258057, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.410259246826172, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.770403861999512, "metrics/particle/momentum/class_1/event/px_bias": -0.8633030652999878, "metrics/particle/momentum/class_1/event/py_bias": -0.3553614914417267, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.2016706466674805, "metrics/particle/momentum/class_1/event/pz_bias": 0.017977207899093628, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.654138565063477, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04185602068901062, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.196482852101326, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19622020423412323, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04051005840301514, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23741240799427032, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03661513328552246, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6934956312179565, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.10225632041692734, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.634225368499756, "metrics/particle/momentum/class_2/accepted/samples": 1229.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9437570571899414, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6589415669441223, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 68022.625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 68022.109375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3715701103210449, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3484418.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07736280560493469, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.195920944213867, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.14805442094802856, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.901468276977539, "metrics/particle/momentum/class_2/assigned/samples": 2624.0, "metrics/particle/momentum/class_2/efficiency": 0.417175829410553, "metrics/particle/momentum/class_2/event/energy_bias": -9.510281562805176, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.219393730163574, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.11582088470459, "metrics/particle/momentum/class_2/event/px_bias": 0.291925311088562, "metrics/particle/momentum/class_2/event/py_bias": -0.07848811149597168, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.617708206176758, "metrics/particle/momentum/class_2/event/pz_bias": -0.4094502031803131, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.73470687866211, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029186153784394264, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14163298904895782, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14178115129470825, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04301133379340172, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18403419852256775, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012325028656050563, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24000562727451324, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0337708480656147, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0578886270523071, "metrics/particle/momentum/class_3/accepted/samples": 11359.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3370821177959442, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5588163137435913, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7160214185714722, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5827100276947021, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17522679269313812, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 33.019142150878906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007980355992913246, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5762284994125366, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.04288715869188309, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7547249794006348, "metrics/particle/momentum/class_3/assigned/samples": 14661.0, "metrics/particle/momentum/class_3/efficiency": 0.737023115158081, "metrics/particle/momentum/class_3/event/energy_bias": 5.493344306945801, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.137566566467285, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.327193737030029, "metrics/particle/momentum/class_3/event/px_bias": -0.4222608208656311, "metrics/particle/momentum/class_3/event/py_bias": 0.18272006511688232, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.497070789337158, "metrics/particle/momentum/class_3/event/pz_bias": -1.4505233764648438, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.843377113342285, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013519919477403164, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10333779454231262, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.057730212807655334, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0016429999377578497, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10797236114740372, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2621929943561554, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.01979440078139305, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9061588644981384, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11872098594903946, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.7279671430587769, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.5477780699729919, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.4880211651325226, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.509277820587158, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38411834836006165, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.051293857395648956, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2576467990875244, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.4680907130241394, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8225290775299072, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.128798007965088, "metrics/particle/momentum/class_4/event/px_bias": 0.1782156378030777, "metrics/particle/momentum/class_4/event/py_bias": 0.03867923468351364, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2248421907424927, "metrics/particle/momentum/class_4/event/pz_bias": 0.2385718822479248, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.728757858276367, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007974796928465366, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023647449910640717, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021085163578391075, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.004728894680738449, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.062053438276052475, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.24735693633556366, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.15588311851024628, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8904879093170166, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.008740234188735485, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.038508038967847824, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03558647632598877, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0010594743071123958, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11928249895572662, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5614967346191406, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4624105989933014, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6226465702056885, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.39223340153694153, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.106390357017517, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.766310453414917, "metrics/particle/momentum/class_5/event/px_bias": 0.0015052780508995056, "metrics/particle/momentum/class_5/event/py_bias": 0.07919996231794357, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6736781597137451, "metrics/particle/momentum/class_5/event/pz_bias": -0.21001900732517242, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4928200244903564, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3543872833251953, "step_valid_time": 4425.826300144196, "step_total_time": 4427.1806881427765, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982029218733174, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06576156442387071, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2879, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548922056384743, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9533112582781457, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9541010770505385, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046688741721854304, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9532338308457712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9516556291390729, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9524440762220381, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548922056384743, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985042267178609, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08002859275023422, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2113, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2109, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378606302707501, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9462606359158083, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.942041908158716, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05373936408419167, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9360852197070573, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9444693237796686, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9402585822559073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378606302707501, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005166899480653, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08416788694453758, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1181, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1175, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9190661478599221, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9233776387802971, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9212168486739469, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0766223612197029, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.914396887159533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9186864738076622, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9165366614664586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9190661478599221} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_26000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..2063d18c7a82b8c5f50d377a6abdb7ce28314182 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2317604699639948, "train": {"Classification_binary": 2.6751480102539062, "Classification": 0.02293584868311882, "Regression_pt": 0.15020428597927094, "Regression_eta": 0.011314730159938335, "Regression_sin_phi": 0.009121693670749664, "Regression_cos_phi": 0.010511319153010845, "Regression_energy": 0.15315461158752441, "Total": 9.480436325073242, "Total_unweighted": 3.0323904994875193}, "train_group": {}, "valid": {"Classification_binary": 2.6306444803873696, "Classification": 0.02128653476635615, "Regression_pt": 0.13741605480511984, "Regression_eta": 0.010908561448256174, "Regression_sin_phi": 0.008250804618000984, "Regression_cos_phi": 0.008512762064735094, "Regression_energy": 0.13985081513722739, "Total": 8.819138844807943, "Total_unweighted": 2.956870013227065, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021852917969226837, "diagnostic/loss/clic_pf/Classification_binary": 2.6439733505249023, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008143141865730286, "diagnostic/loss/clic_pf/Regression_energy": 0.1414896547794342, "diagnostic/loss/clic_pf/Regression_eta": 0.01038212701678276, "diagnostic/loss/clic_pf/Regression_pt": 0.13918036222457886, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007916348055005074, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05367260426282883, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19496722519397736, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0692262277007103, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43362727761268616, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.008256134577095509, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19157658517360687, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07710066437721252, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43807369470596313, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9625300765037537, "metrics/particle/class_1/pid_accuracy": 0.9854817986488342, "metrics/particle/class_2/efficiency": 0.4317771792411804, "metrics/particle/class_2/pid_accuracy": 0.7511099576950073, "metrics/particle/class_3/efficiency": 0.7329668998718262, "metrics/particle/class_3/pid_accuracy": 0.9836793541908264, "metrics/particle/class_4/efficiency": 0.9215116500854492, "metrics/particle/class_4/pid_accuracy": 0.40168243646621704, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.7131147384643555, "metrics/particle/count/bias_mean": -0.7934640645980835, "metrics/particle/count/mae": 3.3019607067108154, "metrics/particle/count/prediction_mean": 69.20587921142578, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.052282292395830154, "metrics/particle/event/energy_response_mean": 1.003677248954773, "metrics/particle/event/met_abs_error": 4.548929214477539, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040426552295684814, "metrics/particle/event/scalar_pt_response_mean": 1.0079693794250488, "metrics/particle/event/vector_pt_closure": 0.04265229031443596, "metrics/particle/matched/delta_eta_abs_mean": 0.00971268955618143, "metrics/particle/matched/delta_phi_abs_mean": 0.010158142074942589, "metrics/particle/matched/delta_r_mean": 0.015827784314751625, "metrics/particle/matched/energy_relative_abs_mean": 0.08407243341207504, "metrics/particle/matched/pid_accuracy": 0.9667031764984131, "metrics/particle/matched/pt_relative_abs_mean": 0.07700661569833755, "metrics/particle/matching/duplicate_fraction": 0.030759787186980247, "metrics/particle/matching/efficiency": 0.8188311457633972, "metrics/particle/matching/f1": 0.8234984874725342, "metrics/particle/matching/purity": 0.8282192945480347, "metrics/particle/matching/target_coverage_dr0p05": 0.794059693813324, "metrics/particle/matching/target_coverage_dr0p10": 0.8565719723701477, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7241617441177368, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8563665151596069, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019226882606744766, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08156900852918625, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07602284103631973, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01632319577038288, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13266955316066742, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003902093041688204, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3459141254425049, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02686767652630806, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2962526082992554, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8457.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2980979382991791, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0655243396759033, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8758996725082397, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7598850727081299, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13172146677970886, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.842827796936035, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013926498591899872, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7784563302993774, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.015815163031220436, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9018263816833496, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10340.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7867708802223206, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.7158464193344116, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.2962646484375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.921443223953247, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.052479714155197144, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07347844541072845, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.320592880249023, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.20635855197906494, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.398664474487305, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028710205107927322, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.095142662525177, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09274589270353317, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011627561412751675, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15105129778385162, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021666666492819786, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9933210611343384, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.030080681666731834, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.345226526260376, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1800.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7270039916038513, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.422961711883545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.405437707901001, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.2451765537261963, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.252145916223526, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 55.694427490234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06795421987771988, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.704622268676758, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.14839228987693787, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.394401550292969, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2796.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5859375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.548635482788086, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.463399410247803, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1457375288009644, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05374760925769806, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.014166874811053276, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.318625450134277, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5430452823638916, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.02109146118164, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013184688985347748, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08441686630249023, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07600279152393341, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.022843176499009132, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13334646821022034, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005790997529402375, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11141639947891235, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.007139505818486214, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5515131950378418, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18995.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14186754822731018, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6708162426948547, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5510007739067078, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46035826206207275, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09283893555402756, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.35889196395874, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029663962777704, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2502894103527069, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0007048644474707544, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3810416460037231, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21575.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8652181625366211, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.1148228645324707, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.212089538574219, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.1747612953186035, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.544491171836853, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.01588350534439087, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9351351261138916, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.20301319658756256, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.329736232757568, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015886887907981873, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08425353467464447, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07703885436058044, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020268050953745842, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1343095749616623, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028374127577990294, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23347900807857513, "metrics/particle/momentum/all/accepted/pz_bias": 0.010552770458161831, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1712791919708252, "metrics/particle/momentum/all/accepted/samples": 29252.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.235539972782135, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8489810824394226, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8777115941047668, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7739032506942749, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11725389957427979, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.373565673828125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011466105468571186, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6053226590156555, "metrics/particle/momentum/all/assigned/pz_bias": -0.006803855765610933, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.061042070388794, "metrics/particle/momentum/all/assigned/samples": 34711.0, "metrics/particle/momentum/all/efficiency": 0.8176659941673279, "metrics/particle/momentum/all/event/energy_bias": -1.1496590375900269, "metrics/particle/momentum/all/event/p3_error_mean": 14.139859199523926, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.6996026039123535, "metrics/particle/momentum/all/event/px_bias": -0.6507185697555542, "metrics/particle/momentum/all/event/py_bias": 0.04342806339263916, "metrics/particle/momentum/all/event/pz_abs_mean": 10.76222038269043, "metrics/particle/momentum/all/event/pz_bias": -0.5463905930519104, "metrics/particle/momentum/all/event/pz_residual_rms": 19.32758140563965, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007988492958247662, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0355614610016346, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.023563707247376442, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007069110404700041, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06542278826236725, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015427337493747473, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12008067220449448, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005893908906728029, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8831269145011902, "metrics/particle/momentum/class_1/accepted/samples": 16205.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.053502652794122696, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.15944652259349823, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.16024117171764374, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.13054287433624268, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.024064963683485985, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.8852670192718506, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047418647445738316, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2652921676635742, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.019141104072332382, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.395845890045166, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9604101181030273, "metrics/particle/momentum/class_1/event/energy_bias": 3.088408946990967, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.727140426635742, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.991632461547852, "metrics/particle/momentum/class_1/event/px_bias": -0.4922821521759033, "metrics/particle/momentum/class_1/event/py_bias": -0.4093075394630432, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.329590797424316, "metrics/particle/momentum/class_1/event/pz_bias": 0.3166704475879669, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.247461318969727, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038191378116607666, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20247183740139008, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20322199165821075, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04892328009009361, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24285927414894104, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03613511472940445, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7270584106445312, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.026397207751870155, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.62896466255188, "metrics/particle/momentum/class_2/accepted/samples": 1273.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8953464031219482, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7491415143013, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7991639971733093, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.30748334527015686, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3590232729911804, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9521846771240234, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07630675286054611, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.059802770614624, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.19432617723941803, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.998447895050049, "metrics/particle/momentum/class_2/assigned/samples": 2621.0, "metrics/particle/momentum/class_2/efficiency": 0.43211132287979126, "metrics/particle/momentum/class_2/event/energy_bias": -9.083968162536621, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.10449504852295, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.177961349487305, "metrics/particle/momentum/class_2/event/px_bias": -0.13922372460365295, "metrics/particle/momentum/class_2/event/py_bias": -0.11874721944332123, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.459760665893555, "metrics/particle/momentum/class_2/event/pz_bias": -0.760651707649231, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.31146812438965, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02502414397895336, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1407349407672882, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14110393822193146, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.047513481229543686, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1833396553993225, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010654354700818658, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22748079895973206, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01579374261200428, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9572428464889526, "metrics/particle/momentum/class_3/accepted/samples": 11263.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33341220021247864, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6454341411590576, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7106229066848755, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5886852741241455, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1837819367647171, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.19228744506836, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0079045994207263, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5640006065368652, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0022882588673382998, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.56880521774292, "metrics/particle/momentum/class_3/assigned/samples": 14675.0, "metrics/particle/momentum/class_3/efficiency": 0.7307941913604736, "metrics/particle/momentum/class_3/event/energy_bias": 5.735385894775391, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.786637306213379, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.2101359367370605, "metrics/particle/momentum/class_3/event/px_bias": -0.23525123298168182, "metrics/particle/momentum/class_3/event/py_bias": 0.4603010416030884, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.153004169464111, "metrics/particle/momentum/class_3/event/pz_bias": -0.30358195304870605, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.617260932922363, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011251517571508884, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12472692131996155, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06168478727340698, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.013440888375043869, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11229164153337479, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26268190145492554, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.05289207026362419, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8955644369125366, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08985117077827454, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.828966736793518, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.543961524963379, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4911903142929077, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.102827072143555, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38761594891548157, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.01179240457713604, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3586794137954712, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.4928792417049408, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7817997932434082, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1265171766281128, "metrics/particle/momentum/class_4/event/px_bias": 0.15102502703666687, "metrics/particle/momentum/class_4/event/py_bias": 0.04025989770889282, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1784238815307617, "metrics/particle/momentum/class_4/event/pz_bias": 0.36831027269363403, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.548125267028809, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0059464662335813046, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022092731669545174, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017946947365999222, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0055351476185023785, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05025637149810791, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2004823535680771, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08945376425981522, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.886951208114624, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.006221636198461056, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03641977906227112, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03245363011956215, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.004863426089286804, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11370247602462769, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5430973768234253, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.43450894951820374, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7235107421875, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3966066539287567, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0487345457077026, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7045606374740601, "metrics/particle/momentum/class_5/event/px_bias": 0.0650135800242424, "metrics/particle/momentum/class_5/event/py_bias": 0.0709218680858612, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6545810699462891, "metrics/particle/momentum/class_5/event/pz_bias": -0.16713766753673553, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5093071460723877, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2722117900848389, "step_valid_time": 4565.521698713303, "step_total_time": 4566.793911218643, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036390484676196, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06425627958788271, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2882, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558872305140962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953042328042328, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544626593806922, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046957671957671955, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9532338308457712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9503968253968254, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9518132141082961, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558872305140962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024862842163462, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08138146818920244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2130, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9471815357301376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9505567928730512, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9488661627389952, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049443207126948774, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9454061251664447, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9487750556792873, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9470875944864384, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9471815357301376, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038033850648767, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08178878767848385, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1178, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9192546583850931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9203264671589584, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08074534161490683, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9167315175097276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9145962732919255, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9156626506024096, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9214007782101167} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_27000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..69e421fd81bc6090b5b61bd09569a789cabf21b6 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2791358726549176, "train": {"Classification_binary": 2.7499005794525146, "Classification": 0.023257363587617874, "Regression_pt": 0.15043002367019653, "Regression_eta": 0.014868187718093395, "Regression_sin_phi": 0.009024306200444698, "Regression_cos_phi": 0.010290159843862057, "Regression_energy": 0.1539449542760849, "Total": 9.709238052368164, "Total_unweighted": 3.111715574748814}, "train_group": {}, "valid": {"Classification_binary": 2.6147754987080893, "Classification": 0.021028863886992138, "Regression_pt": 0.13739627599716187, "Regression_eta": 0.010882275799910227, "Regression_sin_phi": 0.008356808374325434, "Regression_cos_phi": 0.008521314710378647, "Regression_energy": 0.14044235150019327, "Total": 8.80994987487793, "Total_unweighted": 2.941403388977051, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021712545305490494, "diagnostic/loss/clic_pf/Classification_binary": 2.6264991760253906, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008002950809895992, "diagnostic/loss/clic_pf/Regression_energy": 0.1451384425163269, "diagnostic/loss/clic_pf/Regression_eta": 0.010256674140691757, "diagnostic/loss/clic_pf/Regression_pt": 0.1420699656009674, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.00807693600654602, "diagnostic/regression/clic_pf/energy_pred_mean": 0.024201389402151108, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1891101896762848, "diagnostic/regression/clic_pf/energy_residual_mean": 0.03975501284003258, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42524453997612, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.017317764461040497, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18638117611408234, "diagnostic/regression/clic_pf/pt_residual_mean": 0.05152677372097969, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42880624532699585, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.966008722782135, "metrics/particle/class_1/pid_accuracy": 0.9867687225341797, "metrics/particle/class_2/efficiency": 0.4312133491039276, "metrics/particle/class_2/pid_accuracy": 0.7562761306762695, "metrics/particle/class_3/efficiency": 0.7496928572654724, "metrics/particle/class_3/pid_accuracy": 0.9830371737480164, "metrics/particle/class_4/efficiency": 0.930232584476471, "metrics/particle/class_4/pid_accuracy": 0.421875, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.6855983734130859, "metrics/particle/count/bias_mean": -0.5666666626930237, "metrics/particle/count/mae": 3.299999952316284, "metrics/particle/count/prediction_mean": 69.43267822265625, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.051652610301971436, "metrics/particle/event/energy_response_mean": 0.9928128719329834, "metrics/particle/event/met_abs_error": 4.581475734710693, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039598703384399414, "metrics/particle/event/scalar_pt_response_mean": 0.9993516206741333, "metrics/particle/event/vector_pt_closure": 0.040597427636384964, "metrics/particle/matched/delta_eta_abs_mean": 0.010273308493196964, "metrics/particle/matched/delta_phi_abs_mean": 0.010550263337790966, "metrics/particle/matched/delta_r_mean": 0.0165389496833086, "metrics/particle/matched/energy_relative_abs_mean": 0.0816180408000946, "metrics/particle/matched/pid_accuracy": 0.9675712585449219, "metrics/particle/matched/pt_relative_abs_mean": 0.07523518800735474, "metrics/particle/matching/duplicate_fraction": 0.028889600187540054, "metrics/particle/matching/efficiency": 0.8277948498725891, "metrics/particle/matching/f1": 0.8311591148376465, "metrics/particle/matching/purity": 0.8345507979393005, "metrics/particle/matching/target_coverage_dr0p05": 0.7989803552627563, "metrics/particle/matching/target_coverage_dr0p10": 0.8589155673980713, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7343672513961792, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8656383156776428, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01921296864748001, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07956228405237198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07506955415010452, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0027834419161081314, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13086877763271332, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004094046074897051, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.33935457468032837, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0029184932354837656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2405048608779907, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8549.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29122278094291687, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9755721092224121, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9324871301651001, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8064028024673462, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12469862401485443, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.574352264404297, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014948403462767601, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7922845482826233, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03928869962692261, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5979795455932617, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10369.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7953298091888428, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.562402606010437, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.807930946350098, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.9364070892333984, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.014205828309059143, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2061493843793869, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.909712791442871, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.020180359482765198, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.836122512817383, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028837084770202637, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09647789597511292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09356290847063065, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0043861158192157745, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15185946226119995, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020607374608516693, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9808731079101562, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06987831741571426, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.258323907852173, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1844.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7327804565429688, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2008767127990723, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5673.72705078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5673.5517578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23984457552433014, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 301734.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07912398129701614, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.865792989730835, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.0005184134352020919, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.068475723266602, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2831.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6002604365348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.0690033435821533, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.578993320465088, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1902947425842285, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.059338152408599854, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.018666960299015045, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.427022933959961, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.11021559685468674, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.496564865112305, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014286846853792667, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08137807250022888, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07385510951280594, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.006754123140126467, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13027849793434143, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000521920679602772, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1087634265422821, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008329461328685284, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5180474519729614, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19160.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14183351397514343, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6229953765869141, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4591040015220642, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.354255735874176, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08431945741176605, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.249363899230957, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0022274814546108246, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23534590005874634, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004411738365888596, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1463932991027832, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21549.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8727338910102844, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.30111873149871826, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.096246719360352, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.017503261566162, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.014901436865329742, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.06019279360771179, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.951639413833618, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3636701703071594, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.889880657196045, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016619741916656494, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08179498463869095, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07543611526489258, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0049103861674666405, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1318976879119873, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028085135854780674, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22988459467887878, "metrics/particle/momentum/all/accepted/pz_bias": 0.00019580640946514904, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1320732831954956, "metrics/particle/momentum/all/accepted/samples": 29553.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23455525934696198, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.77528315782547, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 462.8013610839844, "metrics/particle/momentum/all/assigned/pt_relative_bias": 462.6843566894531, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10903910547494888, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 86123.84375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012288123369216919, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6158372163772583, "metrics/particle/momentum/all/assigned/pz_bias": 0.008945533074438572, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.373090982437134, "metrics/particle/momentum/all/assigned/samples": 34749.0, "metrics/particle/momentum/all/efficiency": 0.8260796666145325, "metrics/particle/momentum/all/event/energy_bias": -4.330286979675293, "metrics/particle/momentum/all/event/p3_error_mean": 14.339221954345703, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.650045394897461, "metrics/particle/momentum/all/event/px_bias": 0.030230898410081863, "metrics/particle/momentum/all/event/py_bias": 0.16462354362010956, "metrics/particle/momentum/all/event/pz_abs_mean": 11.05531120300293, "metrics/particle/momentum/all/event/pz_bias": 0.27363502979278564, "metrics/particle/momentum/all/event/pz_residual_rms": 20.092029571533203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008989888243377209, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03261436149477959, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022080032154917717, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0015876477118581533, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06335913389921188, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001352348132058978, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11634739488363266, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005127464886754751, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7805807590484619, "metrics/particle/momentum/class_1/accepted/samples": 16268.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04865672439336777, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10140065103769302, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09820086508989334, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06606065481901169, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021335862576961517, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6360132694244385, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005393231753259897, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26178160309791565, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012683994136750698, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.515387535095215, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9641438722610474, "metrics/particle/momentum/class_1/event/energy_bias": 3.028899669647217, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.086310386657715, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.067010402679443, "metrics/particle/momentum/class_1/event/px_bias": -0.8004543781280518, "metrics/particle/momentum/class_1/event/py_bias": -0.13850346207618713, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.727301597595215, "metrics/particle/momentum/class_1/event/pz_bias": 0.7208534479141235, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.289613723754883, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03918580710887909, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1963081806898117, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19744732975959778, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.045005135238170624, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23734237253665924, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03546099364757538, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.664433240890503, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.022255174815654755, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.795630693435669, "metrics/particle/momentum/class_2/accepted/samples": 1269.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8929059505462646, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7160587906837463, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 6128.59716796875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 6128.09326171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3606870174407959, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 313649.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0793893113732338, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0987982749938965, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05974634364247322, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.4104228019714355, "metrics/particle/momentum/class_2/assigned/samples": 2620.0, "metrics/particle/momentum/class_2/efficiency": 0.43075355887413025, "metrics/particle/momentum/class_2/event/energy_bias": -9.49420166015625, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.13203239440918, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.680437088012695, "metrics/particle/momentum/class_2/event/px_bias": 0.4462723135948181, "metrics/particle/momentum/class_2/event/py_bias": -0.1723981499671936, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.68661117553711, "metrics/particle/momentum/class_2/event/pz_bias": 0.5707235932350159, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.195390701293945, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025219088420271873, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1390252709388733, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13888883590698242, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.015247881412506104, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18004389107227325, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013914253795519471, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2314717173576355, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0041001527570188046, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8953065872192383, "metrics/particle/momentum/class_3/accepted/samples": 11499.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3373500406742096, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5500577688217163, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5498909950256348, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4018622636795044, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1674143522977829, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.073959350585938, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008632408455014229, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5889785885810852, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0003204874519724399, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.106339454650879, "metrics/particle/momentum/class_3/assigned/samples": 14712.0, "metrics/particle/momentum/class_3/efficiency": 0.7461069226264954, "metrics/particle/momentum/class_3/event/energy_bias": 3.0960853099823, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.795787811279297, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.116058826446533, "metrics/particle/momentum/class_3/event/px_bias": 0.17304353415966034, "metrics/particle/momentum/class_3/event/py_bias": 0.36562579870224, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.285764694213867, "metrics/particle/momentum/class_3/event/pz_bias": -1.1586459875106812, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.788220405578613, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011868969537317753, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10112069547176361, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05382072180509567, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.002132963389158249, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10154068470001221, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24796274304389954, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.044341906905174255, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9297348260879517, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.0713386982679367, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.4975426197052002, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3671165704727173, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3189983367919922, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.227649688720703, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.29795926809310913, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.02622978202998638, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0100594758987427, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -0.4888714849948883, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.716459035873413, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.099859356880188, "metrics/particle/momentum/class_4/event/px_bias": 0.1352052092552185, "metrics/particle/momentum/class_4/event/py_bias": 0.025576991960406303, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1117801666259766, "metrics/particle/momentum/class_4/event/pz_bias": 0.3109418749809265, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.150985240936279, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0066377813927829266, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.027068093419075012, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.024358712136745453, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0019228867022320628, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07884769886732101, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.24232055246829987, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09544828534126282, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9769178032875061, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.006831665523350239, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03707990050315857, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.034344322979450226, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.002650959650054574, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12556487321853638, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4995674788951874, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35430917143821716, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.631861448287964, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.47219908237457275, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0692991018295288, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.694410502910614, "metrics/particle/momentum/class_5/event/px_bias": 0.07616416364908218, "metrics/particle/momentum/class_5/event/py_bias": 0.08432238548994064, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6883678436279297, "metrics/particle/momentum/class_5/event/pz_bias": -0.17023798823356628, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6844446659088135, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.303227424621582, "step_valid_time": 4707.225254058838, "step_total_time": 4708.528481960297, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985060298095917, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06258016293036317, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3018, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2865, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9532338308457712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952286282306163, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9527598209845848, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04771371769383698, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502487562189055, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9493041749502982, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497762307309796, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9532338308457712, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9967999573031565, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07687350266897719, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400798934753661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9455357142857143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9427999109726241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054464285714285715, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9383044829116733, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.94375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9410193634542622, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400798934753661, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9963631298878681, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07846614945701358, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1271, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1168, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.914396887159533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9244689221085759, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9194053208137715, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07553107789142408, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9089494163424124, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9189614476789929, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9139280125195618, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.914396887159533} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_28000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..41d44c36df15e03ee41a07439b87727cd354992d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3265112753458403, "train": {"Classification_binary": 2.528496265411377, "Classification": 0.020849943161010742, "Regression_pt": 0.1384146362543106, "Regression_eta": 0.010900351218879223, "Regression_sin_phi": 0.008313613012433052, "Regression_cos_phi": 0.008348383009433746, "Regression_energy": 0.13604192435741425, "Total": 8.661181449890137, "Total_unweighted": 2.8513651164248586}, "train_group": {}, "valid": {"Classification_binary": 2.610730012257894, "Classification": 0.021062480906645458, "Regression_pt": 0.13572113712628683, "Regression_eta": 0.010948973397413889, "Regression_sin_phi": 0.008311839153369268, "Regression_cos_phi": 0.008538580189148584, "Regression_energy": 0.1385591725508372, "Total": 8.757370630900065, "Total_unweighted": 2.9338721955815954, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021658971905708313, "diagnostic/loss/clic_pf/Classification_binary": 2.624028205871582, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008147560060024261, "diagnostic/loss/clic_pf/Regression_energy": 0.1411234438419342, "diagnostic/loss/clic_pf/Regression_eta": 0.010287615470588207, "diagnostic/loss/clic_pf/Regression_pt": 0.13841846585273743, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007928909733891487, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03948209807276726, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1905045211315155, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05503571778535843, "diagnostic/regression/clic_pf/energy_residual_rms": 0.428519606590271, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.002423343015834689, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.188139870762825, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06642119586467743, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43465501070022583, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9635239839553833, "metrics/particle/class_1/pid_accuracy": 0.9873948097229004, "metrics/particle/class_2/efficiency": 0.4312133491039276, "metrics/particle/class_2/pid_accuracy": 0.7405857443809509, "metrics/particle/class_3/efficiency": 0.7559219598770142, "metrics/particle/class_3/pid_accuracy": 0.9858857989311218, "metrics/particle/class_4/efficiency": 0.9224806427955627, "metrics/particle/class_4/pid_accuracy": 0.3907563090324402, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.688524603843689, "metrics/particle/count/bias_mean": 0.23856209218502045, "metrics/particle/count/mae": 3.1732025146484375, "metrics/particle/count/prediction_mean": 70.23790740966797, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.048120856285095215, "metrics/particle/event/energy_response_mean": 0.997452974319458, "metrics/particle/event/met_abs_error": 4.435058116912842, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03756282106041908, "metrics/particle/event/scalar_pt_response_mean": 1.0047193765640259, "metrics/particle/event/vector_pt_closure": 0.0402117595076561, "metrics/particle/matched/delta_eta_abs_mean": 0.009442795999348164, "metrics/particle/matched/delta_phi_abs_mean": 0.010778450407087803, "metrics/particle/matched/delta_r_mean": 0.016041120514273643, "metrics/particle/matched/energy_relative_abs_mean": 0.08339665085077286, "metrics/particle/matched/pid_accuracy": 0.9681331515312195, "metrics/particle/matched/pt_relative_abs_mean": 0.07672670483589172, "metrics/particle/matching/duplicate_fraction": 0.028297848999500275, "metrics/particle/matching/efficiency": 0.8292047381401062, "metrics/particle/matching/f1": 0.8277941942214966, "metrics/particle/matching/purity": 0.8263883590698242, "metrics/particle/matching/target_coverage_dr0p05": 0.8041998744010925, "metrics/particle/matching/target_coverage_dr0p10": 0.8629212379455566, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7359359264373779, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8678232431411743, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01782890036702156, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08061923086643219, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07668564468622208, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014549932442605495, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1322094053030014, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0032763865310698748, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.33652621507644653, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.017073513939976692, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.216090202331543, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8546.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2878261208534241, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8112046122550964, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8913657665252686, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7687103748321533, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.129835307598114, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.539041519165039, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014362312853336334, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7493727207183838, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.034040190279483795, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6502256393432617, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10444.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7950506806373596, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7969509363174438, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.35491943359375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8896448612213135, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.25887906551361084, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1766015887260437, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.446731090545654, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.24819813668727875, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.585597038269043, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026529185473918915, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09175284206867218, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08927693217992783, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0070879049599170685, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1434924453496933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01828254759311676, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9169591665267944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07567191123962402, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.958740472793579, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1805.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7658790946006775, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0781351327896118, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 67522.703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 67522.5234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25836819410324097, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3615951.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07357043027877808, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7712793350219727, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0029564667493104935, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.538429260253906, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2868.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5875651240348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.9348864555358887, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.709373474121094, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0350490808486938, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.042921267449855804, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.16289624571800232, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.572089195251465, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.2624577283859253, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.155609130859375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014286398887634277, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08376114815473557, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07543391734361649, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014034098945558071, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13103748857975006, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005723204812966287, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11150261759757996, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0041284579783678055, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5601311326026917, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19220.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14317630231380463, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5368624329566956, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.39599695801734924, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.29122239351272583, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08794352412223816, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.338076591491699, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002399298595264554, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23426415026187897, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007679731585085392, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1961017847061157, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21673.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8754668831825256, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0422106981277466, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.007440090179443, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.029461860656738, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.40084555745124817, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.1974044293165207, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.808777332305908, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3122572600841522, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.5658955574035645, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016057472676038742, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08334094285964966, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07664063572883606, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013759180903434753, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13216917216777802, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024348178412765265, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22569897770881653, "metrics/particle/momentum/all/accepted/pz_bias": 0.012236549519002438, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0796602964401245, "metrics/particle/momentum/all/accepted/samples": 29571.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23740623891353607, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6631337404251099, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5535.88720703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5535.77197265625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11442046612501144, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1035312.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011805059388279915, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5960178375244141, "metrics/particle/momentum/all/assigned/pz_bias": 0.015161861665546894, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9932658672332764, "metrics/particle/momentum/all/assigned/samples": 34985.0, "metrics/particle/momentum/all/efficiency": 0.8265827894210815, "metrics/particle/momentum/all/event/energy_bias": -2.689626693725586, "metrics/particle/momentum/all/event/p3_error_mean": 13.603431701660156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.632260799407959, "metrics/particle/momentum/all/event/px_bias": -0.7026458382606506, "metrics/particle/momentum/all/event/py_bias": 0.5369023084640503, "metrics/particle/momentum/all/event/pz_abs_mean": 10.280566215515137, "metrics/particle/momentum/all/event/pz_bias": 0.32651689648628235, "metrics/particle/momentum/all/event/pz_residual_rms": 17.68534278869629, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007761094253510237, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.035309724509716034, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.024026766419410706, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.009264122694730759, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06364113837480545, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012332737678661942, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11662209033966064, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.011765784583985806, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7896837592124939, "metrics/particle/momentum/class_1/accepted/samples": 16217.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04984363541007042, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1288444697856903, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1436850130558014, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11608043313026428, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.024537695571780205, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.196043491363525, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004800853319466114, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2588384449481964, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02722647227346897, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.288926124572754, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9611213207244873, "metrics/particle/momentum/class_1/event/energy_bias": 4.56412410736084, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.442845344543457, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.016356468200684, "metrics/particle/momentum/class_1/event/px_bias": -0.40650737285614014, "metrics/particle/momentum/class_1/event/py_bias": -0.24400757253170013, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.962685585021973, "metrics/particle/momentum/class_1/event/pz_bias": 0.4564756453037262, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.886085510253906, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03816483914852142, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.201560840010643, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20073066651821136, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05394330248236656, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24078063666820526, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028685258701443672, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5634671449661255, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04371084272861481, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.2417280673980713, "metrics/particle/momentum/class_2/accepted/samples": 1255.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.940819501876831, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6743382811546326, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 71935.3046875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 71934.765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37704309821128845, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3732284.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07838038355112076, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1138370037078857, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.06204238533973694, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.356669902801514, "metrics/particle/momentum/class_2/assigned/samples": 2692.0, "metrics/particle/momentum/class_2/efficiency": 0.4260013699531555, "metrics/particle/momentum/class_2/event/energy_bias": -10.906085968017578, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.706232070922852, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.799257278442383, "metrics/particle/momentum/class_2/event/px_bias": -0.002696346491575241, "metrics/particle/momentum/class_2/event/py_bias": -0.2939341068267822, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.312536239624023, "metrics/particle/momentum/class_2/event/pz_bias": -0.08172222971916199, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.2640323638916, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02554137073457241, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13750602304935455, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13791108131408691, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027264658361673355, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1794162094593048, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013802621979266405, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2316731959581375, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008116117678582668, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9645333290100098, "metrics/particle/momentum/class_3/accepted/samples": 11592.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32773613929748535, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2452352046966553, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3576892614364624, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2155382633209229, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1709350049495697, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.753039360046387, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00799892470240593, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5269967913627625, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.021328028291463852, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.254504919052124, "metrics/particle/momentum/class_3/assigned/samples": 14877.0, "metrics/particle/momentum/class_3/efficiency": 0.7521411776542664, "metrics/particle/momentum/class_3/event/energy_bias": 4.729006767272949, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.832806587219238, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.123998641967773, "metrics/particle/momentum/class_3/event/px_bias": -0.4765945076942444, "metrics/particle/momentum/class_3/event/py_bias": 0.8583287000656128, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.318009853363037, "metrics/particle/momentum/class_3/event/pz_bias": -0.24124819040298462, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.475919723510742, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011968914419412613, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12465747445821762, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06971704214811325, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.020686250180006027, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1191205158829689, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3206773102283478, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08417696505784988, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9673181176185608, "metrics/particle/momentum/class_4/accepted/samples": 331.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.14189386367797852, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8674769401550293, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.529456377029419, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4646224975585938, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.060361862182617, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4386614263057709, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.004895507823675871, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2181079387664795, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9143646359443665, "metrics/particle/momentum/class_4/event/energy_bias": -0.6052046418190002, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8808941841125488, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.215924620628357, "metrics/particle/momentum/class_4/event/px_bias": 0.012844832614064217, "metrics/particle/momentum/class_4/event/py_bias": 0.12005184590816498, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2102340459823608, "metrics/particle/momentum/class_4/event/pz_bias": 0.3800892233848572, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.388774394989014, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005907611921429634, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020836003124713898, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017270473763346672, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008160664699971676, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.03991248458623886, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1649618297815323, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03272966295480728, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6501992344856262, "metrics/particle/momentum/class_5/accepted/samples": 176.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.026995915919542313, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.05032983794808388, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04667696729302406, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.021154988557100296, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.032967034727334976, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.17478898167610168, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5669572353363037, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4254625737667084, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.20039176940918, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9670329689979553, "metrics/particle/momentum/class_5/event/energy_bias": -0.4714663326740265, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.955114483833313, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6723780035972595, "metrics/particle/momentum/class_5/event/px_bias": 0.17030751705169678, "metrics/particle/momentum/class_5/event/py_bias": 0.09646344184875488, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5598853230476379, "metrics/particle/momentum/class_5/event/pz_bias": -0.18707753717899323, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.328129768371582, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4107186794281006, "step_valid_time": 4847.680076599121, "step_total_time": 4849.090795516968, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025124739579925, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06252640499430695, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2881, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9555555555555556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952396694214876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9539735099337748, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047603305785123964, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535655058043118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9504132231404959, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9519867549668874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9555555555555556, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013181287850887, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07575296954434008, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2238, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2131, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526362823949955, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494544644845246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04736371760500447, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9458499778073679, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9521894548704201, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490091293698508, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9462938304482912, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0058824317537, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08040652057158859, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9198443579766536, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9256068911511355, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9227166276346604, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07439310884886452, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9216914643696162, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9188134270101483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9198443579766536} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_29000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..be487de95602430b22ff71b6c772bdc9867160a1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3738866780367633, "train": {"Classification_binary": 2.561861991882324, "Classification": 0.021268034353852272, "Regression_pt": 0.1254187822341919, "Regression_eta": 0.010743373073637486, "Regression_sin_phi": 0.010217171162366867, "Regression_cos_phi": 0.008824761025607586, "Regression_energy": 0.12661895155906677, "Total": 8.561832427978516, "Total_unweighted": 2.864953065291047}, "train_group": {}, "valid": {"Classification_binary": 2.5890115102132163, "Classification": 0.02077191447218259, "Regression_pt": 0.1354659696420034, "Regression_eta": 0.010704572002092997, "Regression_sin_phi": 0.00837375596165657, "Regression_cos_phi": 0.008534658079346022, "Regression_energy": 0.13846728205680847, "Total": 8.715262095133463, "Total_unweighted": 2.911329662427306, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021479878574609756, "diagnostic/loss/clic_pf/Classification_binary": 2.604159116744995, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00810250360518694, "diagnostic/loss/clic_pf/Regression_energy": 0.14147770404815674, "diagnostic/loss/clic_pf/Regression_eta": 0.010138960555195808, "diagnostic/loss/clic_pf/Regression_pt": 0.13856977224349976, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008019530214369297, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04144364595413208, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19051051139831543, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05699727311730385, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4256923496723175, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.00469350628554821, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18626394867897034, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06415103375911713, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43071189522743225, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9635835886001587, "metrics/particle/class_1/pid_accuracy": 0.9873130321502686, "metrics/particle/class_2/efficiency": 0.43425801396369934, "metrics/particle/class_2/pid_accuracy": 0.7520124912261963, "metrics/particle/class_3/efficiency": 0.7498437166213989, "metrics/particle/class_3/pid_accuracy": 0.9847365617752075, "metrics/particle/class_4/efficiency": 0.9205426573753357, "metrics/particle/class_4/pid_accuracy": 0.3831579089164734, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6843177080154419, "metrics/particle/count/bias_mean": 0.12483660131692886, "metrics/particle/count/mae": 3.1549019813537598, "metrics/particle/count/prediction_mean": 70.12418365478516, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05097295716404915, "metrics/particle/event/energy_response_mean": 1.009746789932251, "metrics/particle/event/met_abs_error": 4.407883167266846, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039581671357154846, "metrics/particle/event/scalar_pt_response_mean": 1.0090184211730957, "metrics/particle/event/vector_pt_closure": 0.04048024117946625, "metrics/particle/matched/delta_eta_abs_mean": 0.008764048106968403, "metrics/particle/matched/delta_phi_abs_mean": 0.011662675999104977, "metrics/particle/matched/delta_r_mean": 0.016246475279331207, "metrics/particle/matched/energy_relative_abs_mean": 0.08499327301979065, "metrics/particle/matched/pid_accuracy": 0.9679073095321655, "metrics/particle/matched/pt_relative_abs_mean": 0.0757850706577301, "metrics/particle/matching/duplicate_fraction": 0.028586074709892273, "metrics/particle/matching/efficiency": 0.8268611431121826, "metrics/particle/matching/f1": 0.8261244893074036, "metrics/particle/matching/purity": 0.8253891468048096, "metrics/particle/matching/target_coverage_dr0p05": 0.7958711385726929, "metrics/particle/matching/target_coverage_dr0p10": 0.8600173592567444, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7364867925643921, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.867141604423523, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019491292536258698, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08127705752849579, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07463506609201431, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011427857913076878, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13004662096500397, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0034113633446395397, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32249966263771057, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.029410522431135178, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1819546222686768, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8501.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2903253734111786, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8427468538284302, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8386884331703186, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7168222069740295, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12908481061458588, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.000035285949707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012937230058014393, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7428872585296631, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05670418590307236, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2681736946105957, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10435.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7908642888069153, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.14770768582820892, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.1821818351745605, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7313785552978516, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.02544451504945755, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.04041871428489685, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.313937187194824, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1073005199432373, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.232240676879883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029758255928754807, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09193192422389984, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08911266177892685, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017416512593626976, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14518356323242188, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018539326265454292, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9817690849304199, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09895680099725723, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.982593297958374, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1780.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7640227675437927, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2255829572677612, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10922.083984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10921.9228515625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26330435276031494, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 585502.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07756521552801132, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0107524394989014, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.13221605122089386, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.845406532287598, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2875.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5794270634651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.024701565504074097, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.401885032653809, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1995819807052612, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0023376569151878357, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.17889316380023956, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.241493225097656, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.10424543917179108, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 18.102840423583984, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013595710508525372, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08590252697467804, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07469833642244339, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014194910414516926, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13070659339427948, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007289768545888364, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10842724144458771, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0009570799302309752, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5741122364997864, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19205.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14276117086410522, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5794912576675415, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4457336366176605, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34590116143226624, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08708694577217102, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.973412990570068, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029075134079903364, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23691244423389435, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0035435387399047613, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3003389835357666, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21668.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8747836351394653, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.4421650469303131, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.081525802612305, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.038378715515137, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3745480179786682, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.27469009160995483, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9123754501342773, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.33783596754074097, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.2748823165893555, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016271138563752174, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08493296802043915, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07555025815963745, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01359163224697113, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13143709301948547, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002577494364231825, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2228672355413437, "metrics/particle/momentum/all/accepted/pz_bias": 0.015076398849487305, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.254967212677002, "metrics/particle/momentum/all/accepted/samples": 29486.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23784835636615753, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.711133599281311, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 898.2619018554688, "metrics/particle/momentum/all/assigned/pt_relative_bias": 898.150390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11410029232501984, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 167861.125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012036137282848358, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6158545017242432, "metrics/particle/momentum/all/assigned/pz_bias": 0.025588879361748695, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.493021011352539, "metrics/particle/momentum/all/assigned/samples": 34978.0, "metrics/particle/momentum/all/efficiency": 0.8242068290710449, "metrics/particle/momentum/all/event/energy_bias": 0.5651711225509644, "metrics/particle/momentum/all/event/p3_error_mean": 13.862922668457031, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.539038181304932, "metrics/particle/momentum/all/event/px_bias": 0.3467658460140228, "metrics/particle/momentum/all/event/py_bias": -0.05537818744778633, "metrics/particle/momentum/all/event/pz_abs_mean": 10.66018295288086, "metrics/particle/momentum/all/event/pz_bias": 1.549381971359253, "metrics/particle/momentum/all/event/pz_residual_rms": 19.53107261657715, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008031676523387432, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03881095349788666, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02309914119541645, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00675039179623127, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0634327083826065, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013568521244451404, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11410827934741974, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.008181768469512463, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8015161156654358, "metrics/particle/momentum/class_1/accepted/samples": 16214.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04941369593143463, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16059455275535583, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15553738176822662, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12662771344184875, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.023589378222823143, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.497279167175293, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004919392988085747, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2569175064563751, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01845419965684414, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.439915657043457, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9609435200691223, "metrics/particle/momentum/class_1/event/energy_bias": 3.715313673019409, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.242904663085938, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.727580547332764, "metrics/particle/momentum/class_1/event/px_bias": -0.19376403093338013, "metrics/particle/momentum/class_1/event/py_bias": -0.410031259059906, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.987879276275635, "metrics/particle/momentum/class_1/event/pz_bias": 0.5307306051254272, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.699870109558105, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037016116082668304, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1975329965353012, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.196802020072937, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.057658590376377106, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23603905737400055, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03095238097012043, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5435808897018433, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.046521157026290894, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3465662002563477, "metrics/particle/momentum/class_2/accepted/samples": 1260.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8925892114639282, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6484789848327637, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11662.7470703125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11662.2451171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36887073516845703, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 605076.1875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07689449936151505, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0677387714385986, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07270180433988571, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.188877105712891, "metrics/particle/momentum/class_2/assigned/samples": 2692.0, "metrics/particle/momentum/class_2/efficiency": 0.4276985824108124, "metrics/particle/momentum/class_2/event/energy_bias": -7.910371780395508, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.58051872253418, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.96122407913208, "metrics/particle/momentum/class_2/event/px_bias": 0.2503393888473511, "metrics/particle/momentum/class_2/event/py_bias": -0.5479052066802979, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.05910873413086, "metrics/particle/momentum/class_2/event/pz_bias": 0.7372596859931946, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.94398307800293, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025882011279463768, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13750873506069183, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13770055770874023, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03139907121658325, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17937590181827545, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013041209895163774, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23158453404903412, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.022257164120674133, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.368852138519287, "metrics/particle/momentum/class_3/accepted/samples": 11502.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33915916085243225, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3386276960372925, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3708465099334717, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2337464094161987, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17289845645427704, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.117446899414062, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008742433972656727, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5870352387428284, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.030812393873929977, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.5143532752990723, "metrics/particle/momentum/class_3/assigned/samples": 14870.0, "metrics/particle/momentum/class_3/efficiency": 0.7463015913963318, "metrics/particle/momentum/class_3/event/energy_bias": 5.838158130645752, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.733850479125977, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.931515693664551, "metrics/particle/momentum/class_3/event/px_bias": 0.05815558880567551, "metrics/particle/momentum/class_3/event/py_bias": 0.8261458277702332, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.3159284591674805, "metrics/particle/momentum/class_3/event/pz_bias": 0.09331896901130676, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.880126953125, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012393975630402565, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.12150579690933228, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05509106442332268, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0034255206119269133, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10162340104579926, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24499468505382538, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.015299943275749683, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8990933895111084, "metrics/particle/momentum/class_4/accepted/samples": 332.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09936483204364777, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3990628719329834, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7619571685791016, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.705574095249176, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.420435428619385, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3478550314903259, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.00168537104036659, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1109130382537842, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9171270728111267, "metrics/particle/momentum/class_4/event/energy_bias": -0.681728720664978, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9031689167022705, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.186977744102478, "metrics/particle/momentum/class_4/event/px_bias": 0.1620337963104248, "metrics/particle/momentum/class_4/event/py_bias": 0.053345270454883575, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2750778198242188, "metrics/particle/momentum/class_4/event/pz_bias": 0.34390562772750854, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.7184739112854, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006152828689664602, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023570191115140915, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017149997875094414, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009395740926265717, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04956864193081856, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17627258598804474, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04390266165137291, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7139863967895508, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019971555098891258, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.038024310022592545, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.031382136046886444, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006912428420037031, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11337686330080032, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5118493437767029, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3823886215686798, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.132038593292236, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3961998522281647, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9727175235748291, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6634781360626221, "metrics/particle/momentum/class_5/event/px_bias": 0.07000111043453217, "metrics/particle/momentum/class_5/event/py_bias": 0.023067083209753036, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6052854657173157, "metrics/particle/momentum/class_5/event/pz_bias": -0.15583297610282898, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4392049312591553, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2002129554748535, "step_valid_time": 4988.261333703995, "step_total_time": 4989.461547136307, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007910458536475, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06262549502916626, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2871, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9538971807628525, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501156260323753, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9520026481297583, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04988437396762471, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9522388059701492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9484638255698712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503475670307845, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9538971807628525, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017250920737393, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07681261217524193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517857142857142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9490318272868907, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048214285714285716, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9449622725255216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9504464285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9476964166481193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9462938304482912, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046631525087546, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07760180666186056, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1178, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221789883268483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9265050820953871, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9243369734789392, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07349491790461297, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9167315175097276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9210320562939797, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9188767550702028, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221789883268483} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_3000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..ef0de0862686413b9c1bd0d2d80a880b65870bea --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14212620807276863, "train": {"Classification_binary": 3.900160551071167, "Classification": 0.03631448745727539, "Regression_pt": 0.2024020552635193, "Regression_eta": 0.012981241568922997, "Regression_sin_phi": 0.011448267847299576, "Regression_cos_phi": 0.012867236509919167, "Regression_energy": 0.20250803232192993, "Total": 12.96965503692627, "Total_unweighted": 4.378681872040033}, "train_group": {}, "valid": {"Classification_binary": 3.812962532043457, "Classification": 0.03582696616649628, "Regression_pt": 0.21267618735631308, "Regression_eta": 0.014740881820519766, "Regression_sin_phi": 0.011611449221769968, "Regression_cos_phi": 0.011857887109120687, "Regression_energy": 0.2152213454246521, "Total": 13.219075520833334, "Total_unweighted": 4.314897249142329, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03696524724364281, "diagnostic/loss/clic_pf/Classification_binary": 3.818769931793213, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.011532926931977272, "diagnostic/loss/clic_pf/Regression_energy": 0.21733558177947998, "diagnostic/loss/clic_pf/Regression_eta": 0.013882957398891449, "diagnostic/loss/clic_pf/Regression_pt": 0.21590866148471832, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.011352124623954296, "diagnostic/regression/clic_pf/energy_pred_mean": 0.0673026368021965, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.24621883034706116, "diagnostic/regression/clic_pf/energy_residual_mean": 0.08285626024007797, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5396497845649719, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0252323467284441, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.24486029148101807, "diagnostic/regression/clic_pf/pt_residual_mean": 0.09407688677310944, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5472577214241028, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9506231546401978, "metrics/particle/class_1/pid_accuracy": 0.9857390522956848, "metrics/particle/class_2/efficiency": 0.2309427112340927, "metrics/particle/class_2/pid_accuracy": 0.3291015625, "metrics/particle/class_3/efficiency": 0.6327621340751648, "metrics/particle/class_3/pid_accuracy": 0.9805498123168945, "metrics/particle/class_4/efficiency": 0.9379844665527344, "metrics/particle/class_4/pid_accuracy": 0.05371900647878647, "metrics/particle/class_5/efficiency": 0.9798792600631714, "metrics/particle/class_5/pid_accuracy": 0.30184805393218994, "metrics/particle/count/bias_mean": -0.3960784375667572, "metrics/particle/count/mae": 4.33202600479126, "metrics/particle/count/prediction_mean": 69.603271484375, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.09653805941343307, "metrics/particle/event/energy_response_mean": 1.033684253692627, "metrics/particle/event/met_abs_error": 12.49212646484375, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08314982056617737, "metrics/particle/event/scalar_pt_response_mean": 1.03535795211792, "metrics/particle/event/vector_pt_closure": 0.09023428708314896, "metrics/particle/matched/delta_eta_abs_mean": 0.014470726251602173, "metrics/particle/matched/delta_phi_abs_mean": 0.013099480420351028, "metrics/particle/matched/delta_r_mean": 0.021835213527083397, "metrics/particle/matched/energy_relative_abs_mean": 0.09470011293888092, "metrics/particle/matched/pid_accuracy": 0.9518733620643616, "metrics/particle/matched/pt_relative_abs_mean": 0.08590977638959885, "metrics/particle/matching/duplicate_fraction": 0.04192763939499855, "metrics/particle/matching/efficiency": 0.7533497214317322, "metrics/particle/matching/f1": 0.7554870843887329, "metrics/particle/matching/purity": 0.7576366662979126, "metrics/particle/matching/target_coverage_dr0p05": 0.6886712312698364, "metrics/particle/matching/target_coverage_dr0p10": 0.7923790216445923, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6664301156997681, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8077666163444519, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024452270939946175, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08894097805023193, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08105320483446121, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020383713766932487, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1348147690296173, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004202250391244888, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38194000720977783, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.037973552942276, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2937430143356323, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7377.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4685729146003723, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.9022272825241089, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.0480270385742188, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.9053919315338135, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1976030468940735, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.3946590423584, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.026113221421837807, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2292581796646118, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.15393945574760437, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.032032012939453, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10263.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6862964034080505, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.013191252946853638, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 12.889951705932617, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.635234355926514, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.12470264732837677, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.24649742245674133, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.129972457885742, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.9575878381729126, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.432363510131836, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0300595685839653, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09881054610013962, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09573688358068466, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.022262180224061012, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1539921909570694, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017921147868037224, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9495257139205933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.059933219105005264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8175253868103027, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1395.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.0004380941390991, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.5828781127929688, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.2238078117370605, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.040101528167725, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3431265950202942, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 36.717620849609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10990206897258759, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.355566501617432, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.388266384601593, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 13.520185470581055, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2757.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4541015625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.8646820783615112, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 13.111471176147461, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.0897111892700195, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.24202260375022888, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07447319477796555, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 12.811504364013672, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -5.397929668426514, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 28.7142276763916, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02003820799291134, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09651725739240646, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08696839958429337, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.028680691495537758, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14103135466575623, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009357626549899578, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13472245633602142, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010916815139353275, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6956664323806763, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18167.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.21516966819763184, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1833635568618774, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9199572801589966, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8039460778236389, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12207140028476715, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.34117603302002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005531796254217625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.43398988246917725, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0065516815520823, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.16091775894165, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21512.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.827502965927124, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 7.380218505859375, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 17.212533950805664, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 14.691323280334473, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.05489093065261841, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.8787401914596558, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.668966293334961, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.9475904703140259, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 38.893611907958984, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021765898913145065, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09456131607294083, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08580264449119568, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.026076262816786766, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14006520807743073, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027098259888589382, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24461422860622406, "metrics/particle/momentum/all/accepted/pz_bias": -0.020864304155111313, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0935237407684326, "metrics/particle/momentum/all/accepted/samples": 26939.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3531768321990967, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5087475776672363, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.5988376140594482, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.4695090055465698, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.16216842830181122, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.359935760498047, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.019981466233730316, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.9834403991699219, "metrics/particle/momentum/all/assigned/pz_bias": -0.08083142340183258, "metrics/particle/momentum/all/assigned/pz_residual_rms": 6.76500940322876, "metrics/particle/momentum/all/assigned/samples": 34532.0, "metrics/particle/momentum/all/efficiency": 0.7530118823051453, "metrics/particle/momentum/all/event/energy_bias": 9.231709480285645, "metrics/particle/momentum/all/event/p3_error_mean": 31.57622528076172, "metrics/particle/momentum/all/event/pt_vector_error_mean": 17.54381561279297, "metrics/particle/momentum/all/event/px_bias": -0.42161625623703003, "metrics/particle/momentum/all/event/py_bias": -1.0507644414901733, "metrics/particle/momentum/all/event/pz_abs_mean": 22.061241149902344, "metrics/particle/momentum/all/event/pz_bias": -5.4079270362854, "metrics/particle/momentum/all/event/pz_residual_rms": 49.216407775878906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012180982157588005, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.050420306622982025, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.037124503403902054, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.024815203621983528, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07354828715324402, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015616215532645583, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14884395897388458, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005749627947807312, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8578824400901794, "metrics/particle/momentum/class_1/accepted/samples": 16009.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06103712320327759, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.2030893713235855, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.23195116221904755, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.20614872872829437, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.033485449850559235, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.797791004180908, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005748829338699579, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3808315396308899, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.046324849128723145, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 6.985656261444092, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9487939476966858, "metrics/particle/momentum/class_1/event/energy_bias": 17.32142448425293, "metrics/particle/momentum/class_1/event/p3_error_mean": 18.545995712280273, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 11.225849151611328, "metrics/particle/momentum/class_1/event/px_bias": -2.5417230129241943, "metrics/particle/momentum/class_1/event/py_bias": -0.9539510011672974, "metrics/particle/momentum/class_1/event/pz_abs_mean": 12.590265274047852, "metrics/particle/momentum/class_1/event/pz_bias": 1.9422965049743652, "metrics/particle/momentum/class_1/event/pz_residual_rms": 42.50304412841797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04590778425335884, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2320156842470169, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22824452817440033, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.08223676681518555, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.26897409558296204, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.041358936578035355, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9158111810684204, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.07182539999485016, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.976344108581543, "metrics/particle/momentum/class_2/accepted/samples": 677.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.3937102556228638, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.140945553779602, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.3027641773223877, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6085746884346008, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5437125563621521, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.8940093517303467, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.12614770233631134, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.780494689941406, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.579136073589325, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.632686614990234, "metrics/particle/momentum/class_2/assigned/samples": 2505.0, "metrics/particle/momentum/class_2/efficiency": 0.22980313003063202, "metrics/particle/momentum/class_2/event/energy_bias": -15.25894546508789, "metrics/particle/momentum/class_2/event/p3_error_mean": 25.195270538330078, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 13.306974411010742, "metrics/particle/momentum/class_2/event/px_bias": 1.5859202146530151, "metrics/particle/momentum/class_2/event/py_bias": 0.21410596370697021, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.042709350585938, "metrics/particle/momentum/class_2/event/pz_bias": -4.655561447143555, "metrics/particle/momentum/class_2/event/pz_residual_rms": 28.875640869140625, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03628325089812279, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15858173370361328, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15801550447940826, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03653555363416672, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19985313713550568, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0018489984795451164, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.28182879090309143, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.058771826326847076, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9568132758140564, "metrics/particle/momentum/class_3/accepted/samples": 9735.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5234315395355225, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.119506359100342, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.272446870803833, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.117252826690674, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2502395510673523, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.543973922729492, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.018685830757021904, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 1.04707670211792, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.13772155344486237, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.674149990081787, "metrics/particle/momentum/class_3/assigned/samples": 14610.0, "metrics/particle/momentum/class_3/efficiency": 0.6316506862640381, "metrics/particle/momentum/class_3/event/energy_bias": 13.276288032531738, "metrics/particle/momentum/class_3/event/p3_error_mean": 17.04373550415039, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.960905075073242, "metrics/particle/momentum/class_3/event/px_bias": 0.12323465943336487, "metrics/particle/momentum/class_3/event/py_bias": -0.3351607620716095, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.371838569641113, "metrics/particle/momentum/class_3/event/pz_bias": -2.2552080154418945, "metrics/particle/momentum/class_3/event/pz_residual_rms": 21.789880752563477, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015698635950684547, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09509269148111343, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.054240062832832336, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.010541521944105625, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09395214170217514, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0029325513169169426, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3013615906238556, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.050694894045591354, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.2672027349472046, "metrics/particle/momentum/class_4/accepted/samples": 341.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06285351514816284, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5297107100486755, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.2677348554134369, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.22059446573257446, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.033149171620607376, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 1.6098705530166626, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.350156307220459, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06801050156354904, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3686614036560059, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9419889450073242, "metrics/particle/momentum/class_4/event/energy_bias": -4.754388332366943, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.846731662750244, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.68049955368042, "metrics/particle/momentum/class_4/event/px_bias": 0.38346216082572937, "metrics/particle/momentum/class_4/event/py_bias": 0.27481651306152344, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.5020196437835693, "metrics/particle/momentum/class_4/event/pz_bias": 0.3004283308982849, "metrics/particle/momentum/class_4/event/pz_residual_rms": 8.158690452575684, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009582030586898327, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03906252235174179, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03284135088324547, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009084218181669712, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07814310491085052, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.35846275091171265, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09069395065307617, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.228950023651123, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02577117271721363, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.26118022203445435, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.6953808069229126, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.6557235717773438, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 8.746833801269531, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.7401490211486816, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.46946337819099426, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.8624699115753174, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -1.352668046951294, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.5928421020507812, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.859220027923584, "metrics/particle/momentum/class_5/event/px_bias": 0.02748984843492508, "metrics/particle/momentum/class_5/event/py_bias": -0.25057512521743774, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.6499090194702148, "metrics/particle/momentum/class_5/event/pz_bias": -0.7398825883865356, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.8691182136535645, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.337524652481079, "step_valid_time": 670.442711353302, "step_total_time": 671.7802362442017, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0138494421456499, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16134007244920578, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3004, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2763, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2672, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9164179104477612, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9197736351531292, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9180927064296395, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08022636484687085, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8862354892205638, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8894806924101198, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.887855125436119, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9164179104477612, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0148552292158528, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17548542276029844, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2006, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8903683976919663, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8959356855739169, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8931433659839715, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10406431442608308, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8708388814913449, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8762840553818669, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8735529830810329, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8903683976919663, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0222972768136318, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19126438642923127, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1272, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1073, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8350194552529183, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8435534591194969, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8392647633946031, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15644654088050314, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8124513618677043, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8207547169811321, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8165819319515056, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8350194552529183} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_30000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..4f65365284c6fe1926e6e8dc5ef86847d5166820 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4212620807276861, "train": {"Classification_binary": 2.5576796531677246, "Classification": 0.021692520007491112, "Regression_pt": 0.12651123106479645, "Regression_eta": 0.009543701075017452, "Regression_sin_phi": 0.008233839645981789, "Regression_cos_phi": 0.007755453698337078, "Regression_energy": 0.13113659620285034, "Total": 8.383424758911133, "Total_unweighted": 2.862552994862199}, "train_group": {}, "valid": {"Classification_binary": 2.591421604156494, "Classification": 0.020692372073729832, "Regression_pt": 0.1353729565938314, "Regression_eta": 0.010852000365654627, "Regression_sin_phi": 0.00819446270664533, "Regression_cos_phi": 0.008430459226171175, "Regression_energy": 0.13816526532173157, "Total": 8.69586435953776, "Total_unweighted": 2.913129120444258, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02115362137556076, "diagnostic/loss/clic_pf/Classification_binary": 2.5987789630889893, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008120059967041016, "diagnostic/loss/clic_pf/Regression_energy": 0.14068622887134552, "diagnostic/loss/clic_pf/Regression_eta": 0.010418481193482876, "diagnostic/loss/clic_pf/Regression_pt": 0.1381206065416336, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007901933044195175, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04166804254055023, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19016648828983307, "diagnostic/regression/clic_pf/energy_residual_mean": 0.057221669703722, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4243045449256897, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.003932958468794823, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1867658495903015, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06491156667470932, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42855238914489746, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9645774960517883, "metrics/particle/class_1/pid_accuracy": 0.9869139790534973, "metrics/particle/class_2/efficiency": 0.4322282373905182, "metrics/particle/class_2/pid_accuracy": 0.7521523833274841, "metrics/particle/class_3/efficiency": 0.7492186427116394, "metrics/particle/class_3/pid_accuracy": 0.9848964214324951, "metrics/particle/class_4/efficiency": 0.9224806427955627, "metrics/particle/class_4/pid_accuracy": 0.38025209307670593, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.718367338180542, "metrics/particle/count/bias_mean": -0.0313725508749485, "metrics/particle/count/mae": 3.205228805541992, "metrics/particle/count/prediction_mean": 69.96797180175781, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.050120823085308075, "metrics/particle/event/energy_response_mean": 1.0057469606399536, "metrics/particle/event/met_abs_error": 4.521666049957275, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039075952023267746, "metrics/particle/event/scalar_pt_response_mean": 1.0051220655441284, "metrics/particle/event/vector_pt_closure": 0.04112829267978668, "metrics/particle/matched/delta_eta_abs_mean": 0.010152238421142101, "metrics/particle/matched/delta_phi_abs_mean": 0.010671581141650677, "metrics/particle/matched/delta_r_mean": 0.016519194468855858, "metrics/particle/matched/energy_relative_abs_mean": 0.08399098366498947, "metrics/particle/matched/pid_accuracy": 0.9679539203643799, "metrics/particle/matched/pt_relative_abs_mean": 0.07691847532987595, "metrics/particle/matching/duplicate_fraction": 0.02884606458246708, "metrics/particle/matching/efficiency": 0.826898455619812, "metrics/particle/matching/f1": 0.8270838260650635, "metrics/particle/matching/purity": 0.8272692561149597, "metrics/particle/matching/target_coverage_dr0p05": 0.7992978692054749, "metrics/particle/matching/target_coverage_dr0p10": 0.8616700172424316, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7303616404533386, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8657317161560059, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018730541691184044, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08216738700866699, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07726199924945831, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021021731197834015, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1344727873802185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002694786060601473, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3216499984264374, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.009223305620253086, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1348685026168823, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8535.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28858456015586853, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8532823920249939, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9490643739700317, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8330094814300537, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12988010048866272, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.861194610595703, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013237410224974155, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7379008531570435, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06556451320648193, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5727486610412598, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10425.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7940273284912109, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5682795643806458, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.756467819213867, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.07716965675354, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.05598399415612221, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.43258586525917053, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.741477012634277, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.7784950733184814, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.361343383789062, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026643574237823486, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1001950055360794, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09896144270896912, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02965037152171135, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1597520112991333, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018722467124462128, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9309096932411194, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1566968411207199, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0639657974243164, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1816.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7283027172088623, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2662532329559326, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 58915.42578125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 58915.26953125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26961982250213623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3154439.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07673526555299759, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8589799404144287, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.21884630620479584, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.86571979522705, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2867.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5911458134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2129685878753662, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.627958297729492, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0706801414489746, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03928123787045479, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08315014094114304, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.486083507537842, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3776529133319855, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.89159870147705, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014706352725625038, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08293424546718597, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07432221621274948, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011592158116400242, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1310121864080429, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005216483841650188, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10870976746082306, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015211760997772217, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5381745100021362, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19170.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14764238893985748, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6021080613136292, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.42041370272636414, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3167414963245392, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08516547083854675, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.9608561992645264, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029622772708535194, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24243700504302979, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012318668887019157, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.350791573524475, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21605.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.873189389705658, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.36696380376815796, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.313986778259277, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.105636119842529, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.28880244493484497, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.38210663199424744, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2007250785827637, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2785751223564148, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.61294412612915, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016604134812951088, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08377435058355331, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07668784260749817, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015429262071847916, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.133954718708992, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002269570715725422, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2208523154258728, "metrics/particle/momentum/all/accepted/pz_bias": 0.002427862025797367, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.066742181777954, "metrics/particle/momentum/all/accepted/samples": 29521.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2374516874551773, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7317065596580505, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4840.8037109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4840.69140625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11367738991975784, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 904153.0625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012092730030417442, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.605414867401123, "metrics/particle/momentum/all/assigned/pz_bias": 0.02993946708738804, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0926589965820312, "metrics/particle/momentum/all/assigned/samples": 34897.0, "metrics/particle/momentum/all/efficiency": 0.8251851797103882, "metrics/particle/momentum/all/event/energy_bias": -0.27772513031959534, "metrics/particle/momentum/all/event/p3_error_mean": 13.555452346801758, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.481240749359131, "metrics/particle/momentum/all/event/px_bias": -0.3055051863193512, "metrics/particle/momentum/all/event/py_bias": 0.8978426456451416, "metrics/particle/momentum/all/event/pz_abs_mean": 10.361467361450195, "metrics/particle/momentum/all/event/pz_bias": 1.434723138809204, "metrics/particle/momentum/all/event/pz_residual_rms": 18.006845474243164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008050372824072838, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03254503756761551, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021105697378516197, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.000351249793311581, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.060618139803409576, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011692307889461517, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10842529684305191, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.006550562102347612, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7664505243301392, "metrics/particle/momentum/class_1/accepted/samples": 16250.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05196651071310043, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13984504342079163, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1428922414779663, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10874379426240921, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022111564874649048, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.848087310791016, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004683146718889475, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24002458155155182, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0017324148211628199, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2764759063720703, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.9630771279335022, "metrics/particle/momentum/class_1/event/energy_bias": 2.551419973373413, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.544551849365234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.015460014343262, "metrics/particle/momentum/class_1/event/px_bias": -0.7112649083137512, "metrics/particle/momentum/class_1/event/py_bias": -0.39830282330513, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.130557060241699, "metrics/particle/momentum/class_1/event/pz_bias": -0.09950272738933563, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.598475456237793, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03848007321357727, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19616083800792694, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19738078117370605, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04518219083547592, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.235492542386055, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02936507947742939, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5334938764572144, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08007949590682983, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3604202270507812, "metrics/particle/momentum/class_2/accepted/samples": 1260.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9256096482276917, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6898529529571533, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 62883.44140625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 62882.9296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3708116114139557, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3258989.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07706627249717712, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1188371181488037, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2410466969013214, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.673243522644043, "metrics/particle/momentum/class_2/assigned/samples": 2686.0, "metrics/particle/momentum/class_2/efficiency": 0.4276985824108124, "metrics/particle/momentum/class_2/event/energy_bias": -8.622724533081055, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.194415092468262, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.731832504272461, "metrics/particle/momentum/class_2/event/px_bias": -0.04200801998376846, "metrics/particle/momentum/class_2/event/py_bias": 0.2933131754398346, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.771711349487305, "metrics/particle/momentum/class_2/event/pz_bias": 0.7959573268890381, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.525455474853516, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026607360690832138, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14440017938613892, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14362482726573944, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04404372721910477, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18560168147087097, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000956771313212812, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23522038757801056, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008967154659330845, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9068391919136047, "metrics/particle/momentum/class_3/accepted/samples": 11497.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3297674357891083, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4150832891464233, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4799708127975464, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3492788076400757, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1738748550415039, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.215213775634766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009122854098677635, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5733955502510071, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03189629316329956, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4011056423187256, "metrics/particle/momentum/class_3/assigned/samples": 14798.0, "metrics/particle/momentum/class_3/efficiency": 0.7459771633148193, "metrics/particle/momentum/class_3/event/energy_bias": 6.595648288726807, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.746479034423828, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.902745246887207, "metrics/particle/momentum/class_3/event/px_bias": 0.14880065619945526, "metrics/particle/momentum/class_3/event/py_bias": 0.9431802034378052, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.397768497467041, "metrics/particle/momentum/class_3/event/pz_bias": 0.5123260021209717, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.247319221496582, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011227102018892765, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09803617745637894, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05254163593053818, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.001075403532013297, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10183092206716537, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24643339216709137, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0050506568513810635, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8694188594818115, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11702611297369003, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0366532802581787, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7241718173027039, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6651485562324524, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.965538024902344, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3559759259223938, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08958442509174347, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1651712656021118, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.502324104309082, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8432646989822388, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.126578688621521, "metrics/particle/momentum/class_4/event/px_bias": 0.17227023839950562, "metrics/particle/momentum/class_4/event/py_bias": 0.0047981347888708115, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.254857063293457, "metrics/particle/momentum/class_4/event/pz_bias": 0.3761378228664398, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.719943046569824, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006685991771519184, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022324031218886375, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018684975802898407, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003930205479264259, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.058115921914577484, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.21650001406669617, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.13583683967590332, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7861721515655518, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.007028729189187288, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.036811504513025284, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.033061638474464417, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00899814534932375, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1173705980181694, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.47813692688941956, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.39258021116256714, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6300406455993652, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.29974520206451416, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9869471788406372, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6690334677696228, "metrics/particle/momentum/class_5/event/px_bias": 0.1266968697309494, "metrics/particle/momentum/class_5/event/py_bias": 0.054853878915309906, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6213460564613342, "metrics/particle/momentum/class_5/event/pz_bias": -0.15019528567790985, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4386279582977295, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4296329021453857, "step_valid_time": 5128.047752857208, "step_total_time": 5129.477385997772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997549582022816, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0628506132399087, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552238805970149, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517514871116987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9534845224300612, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04824851288830139, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535655058043118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9500991407799074, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.951829167356398, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552238805970149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009247409920823, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08081668511373019, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2124, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9427430093209055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9477911646586346, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.945260347129506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05220883534136546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400798934753661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9451137884872824, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9425901201602136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9427430093209055, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013100622095805, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0758933671187153, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1172, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9202501954652071, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9180967238689548, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0797498045347928, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9120622568093385, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9163408913213448, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9141965678627145, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9159533073929961} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_31000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a3d5fb98aa31418bac2fd643d3b1991860a158f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4686374834186091, "train": {"Classification_binary": 2.510075807571411, "Classification": 0.019140100106596947, "Regression_pt": 0.12189463526010513, "Regression_eta": 0.009449509903788567, "Regression_sin_phi": 0.009725472889840603, "Regression_cos_phi": 0.009625671431422234, "Regression_energy": 0.1218143180012703, "Total": 8.271988868713379, "Total_unweighted": 2.801725515164435}, "train_group": {}, "valid": {"Classification_binary": 2.569938818613688, "Classification": 0.02046821638941765, "Regression_pt": 0.13393055399258932, "Regression_eta": 0.010704422990481058, "Regression_sin_phi": 0.008173416058222452, "Regression_cos_phi": 0.008371728161970774, "Regression_energy": 0.13681456446647644, "Total": 8.616446177164713, "Total_unweighted": 2.888401720672846, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021147819235920906, "diagnostic/loss/clic_pf/Classification_binary": 2.5757908821105957, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007963474839925766, "diagnostic/loss/clic_pf/Regression_energy": 0.1393505036830902, "diagnostic/loss/clic_pf/Regression_eta": 0.010062694549560547, "diagnostic/loss/clic_pf/Regression_pt": 0.13668915629386902, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007922171615064144, "diagnostic/regression/clic_pf/energy_pred_mean": 0.042612068355083466, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18865060806274414, "diagnostic/regression/clic_pf/energy_residual_mean": 0.058165691792964935, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4259962737560272, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0010878340108320117, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18615558743476868, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06775670498609543, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4309828579425812, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9645575881004333, "metrics/particle/class_1/pid_accuracy": 0.9866045713424683, "metrics/particle/class_2/efficiency": 0.4333558976650238, "metrics/particle/class_2/pid_accuracy": 0.7514962553977966, "metrics/particle/class_3/efficiency": 0.7478607892990112, "metrics/particle/class_3/pid_accuracy": 0.9848978519439697, "metrics/particle/class_4/efficiency": 0.9273256063461304, "metrics/particle/class_4/pid_accuracy": 0.4263322949409485, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7265306115150452, "metrics/particle/count/bias_mean": -0.6882352828979492, "metrics/particle/count/mae": 3.260784387588501, "metrics/particle/count/prediction_mean": 69.31111145019531, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05058169364929199, "metrics/particle/event/energy_response_mean": 1.0070583820343018, "metrics/particle/event/met_abs_error": 4.80131721496582, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039464741945266724, "metrics/particle/event/scalar_pt_response_mean": 1.0066889524459839, "metrics/particle/event/vector_pt_closure": 0.04193999990820885, "metrics/particle/matched/delta_eta_abs_mean": 0.008361044339835644, "metrics/particle/matched/delta_phi_abs_mean": 0.010944155976176262, "metrics/particle/matched/delta_r_mean": 0.015331383794546127, "metrics/particle/matched/energy_relative_abs_mean": 0.08254127204418182, "metrics/particle/matched/pid_accuracy": 0.9682299494743347, "metrics/particle/matched/pt_relative_abs_mean": 0.07597517222166061, "metrics/particle/matching/duplicate_fraction": 0.02794070541858673, "metrics/particle/matching/efficiency": 0.8264409303665161, "metrics/particle/matching/f1": 0.8305238485336304, "metrics/particle/matching/purity": 0.8346472382545471, "metrics/particle/matching/target_coverage_dr0p05": 0.8013426661491394, "metrics/particle/matching/target_coverage_dr0p10": 0.8592984080314636, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7335549592971802, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8636308312416077, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017417272552847862, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08145798742771149, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07690572738647461, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01826935075223446, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13350042700767517, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003163074143230915, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3389514982700348, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.016356589272618294, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2118946313858032, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8536.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29158905148506165, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9393926858901978, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.003973364830017, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.88954097032547, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12724997103214264, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.09058666229248, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0139570701867342, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7572445869445801, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05016950145363808, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6378421783447266, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10389.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7941203713417053, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.03699587285518646, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.124974727630615, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.759342908859253, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.0011641308665275574, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.24358677864074707, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.232358455657959, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.42706888914108276, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.175600051879883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026394307613372803, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09368035942316055, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0917811170220375, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.025504808872938156, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14728911221027374, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01934770680963993, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9100983738899231, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08207369595766068, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.838895559310913, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1809.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7113704681396484, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0923892259597778, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.691999673843384, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.5402510166168213, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2606081962585449, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 39.42578887939453, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07390382140874863, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6961121559143066, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03663037344813347, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.200754165649414, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2828.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5888671875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.029839038848877, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.920182228088379, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1132084131240845, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.002916347235441208, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03641003742814064, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.753509998321533, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.8770023584365845, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.609373092651367, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01347963884472847, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08269980549812317, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07483924180269241, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013681906275451183, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1313857138156891, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005212677060626447, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10608858615159988, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.012544803321361542, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5245996117591858, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19184.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14092957973480225, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6308528184890747, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4648366868495941, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3652447760105133, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.082872673869133, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.352583408355713, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025084777735173702, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2357122153043747, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.005404970608651638, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3493143320083618, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21527.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8738270998001099, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.16742956638336182, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.201293468475342, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.147998332977295, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.7097476720809937, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1838575601577759, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.9641475677490234, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.11648469418287277, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.607614040374756, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015409073792397976, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08301352709531784, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07647449523210526, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.0157322995364666, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13302494585514069, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002438280964270234, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22265775501728058, "metrics/particle/momentum/all/accepted/pz_bias": 0.0016062736976891756, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0474143028259277, "metrics/particle/momentum/all/accepted/samples": 29529.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23241034150123596, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7606779932975769, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8073276877403259, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6990529894828796, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1106090247631073, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.45551586151123, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011743035167455673, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.591923713684082, "metrics/particle/momentum/all/assigned/pz_bias": 0.014634148217737675, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9466419219970703, "metrics/particle/momentum/all/assigned/samples": 34744.0, "metrics/particle/momentum/all/efficiency": 0.8254088163375854, "metrics/particle/momentum/all/event/energy_bias": -0.8254135847091675, "metrics/particle/momentum/all/event/p3_error_mean": 13.8731689453125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.625312805175781, "metrics/particle/momentum/all/event/px_bias": 0.7079954743385315, "metrics/particle/momentum/all/event/py_bias": -1.463854432106018, "metrics/particle/momentum/all/event/pz_abs_mean": 10.66838264465332, "metrics/particle/momentum/all/event/pz_bias": 1.1875865459442139, "metrics/particle/momentum/all/event/pz_residual_rms": 18.56478500366211, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007751076482236385, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03423148766160011, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.023167861625552177, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007909047417342663, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06319696456193924, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010468624532222748, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11124914139509201, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.008105816319584846, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.758838415145874, "metrics/particle/momentum/class_1/accepted/samples": 16239.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.047850776463747025, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13352878391742706, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14143399894237518, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11408300697803497, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02364724688231945, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.54515266418457, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004326438531279564, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24931877851486206, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012724876403808594, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.355278968811035, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.962425172328949, "metrics/particle/momentum/class_1/event/energy_bias": 4.501359462738037, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.915655136108398, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.119320869445801, "metrics/particle/momentum/class_1/event/px_bias": -0.5930168628692627, "metrics/particle/momentum/class_1/event/py_bias": -0.5038251876831055, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.4332098960876465, "metrics/particle/momentum/class_1/event/pz_bias": 0.1931748390197754, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.328227996826172, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03708591312170029, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19895415008068085, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20031850039958954, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.046129632741212845, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23983487486839294, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.033096928149461746, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6256120204925537, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.01176709309220314, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3681962490081787, "metrics/particle/momentum/class_2/accepted/samples": 1269.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9218628406524658, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6907950043678284, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7727853655815125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.26956307888031006, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.363808810710907, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3663554191589355, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07928679883480072, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0530011653900146, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.019961444661021233, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.890094757080078, "metrics/particle/momentum/class_2/assigned/samples": 2636.0, "metrics/particle/momentum/class_2/efficiency": 0.43075355887413025, "metrics/particle/momentum/class_2/event/energy_bias": -9.669975280761719, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.865530967712402, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.997457027435303, "metrics/particle/momentum/class_2/event/px_bias": 0.7414314150810242, "metrics/particle/momentum/class_2/event/py_bias": -1.0958170890808105, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.384981155395508, "metrics/particle/momentum/class_2/event/pz_bias": 0.6695709824562073, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.671873092651367, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.024063127115368843, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1392294466495514, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13939324021339417, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03373372182250023, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18180625140666962, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011297471355646849, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22374464571475983, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.018233168870210648, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8486986756324768, "metrics/particle/momentum/class_3/accepted/samples": 11507.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3262532651424408, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4707367420196533, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5683307647705078, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4356576204299927, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1675175279378891, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.809730529785156, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008440542034804821, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5491564273834229, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019972512498497963, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3208582401275635, "metrics/particle/momentum/class_3/assigned/samples": 14691.0, "metrics/particle/momentum/class_3/efficiency": 0.7466260194778442, "metrics/particle/momentum/class_3/event/energy_bias": 5.034045219421387, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.909612655639648, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.165574073791504, "metrics/particle/momentum/class_3/event/px_bias": 0.43296757340431213, "metrics/particle/momentum/class_3/event/py_bias": -0.03569244593381882, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.31721830368042, "metrics/particle/momentum/class_3/event/pz_bias": 0.24576136469841003, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.488245964050293, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012190120294690132, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11002524942159653, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06132730469107628, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.014879226684570312, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10838869959115982, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2951664328575134, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.017620598897337914, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9143372178077698, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09980051219463348, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.0473849773406982, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6006371974945068, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5471948385238647, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.142169952392578, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3945317566394806, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.004083114210516214, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.183003306388855, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.5066867470741272, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7991710901260376, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1552032232284546, "metrics/particle/momentum/class_4/event/px_bias": 0.09611047804355621, "metrics/particle/momentum/class_4/event/py_bias": 0.10129939764738083, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1666102409362793, "metrics/particle/momentum/class_4/event/pz_bias": 0.28866031765937805, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.308536052703857, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006137541960924864, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021722476929426193, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017888782545924187, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008366113528609276, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04682047292590141, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17741024494171143, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05558483675122261, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6654727458953857, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04578687250614166, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03999941051006317, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.035921964794397354, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.005897307302802801, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13383954763412476, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5541284680366516, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.27920013666152954, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.253990173339844, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.1841561496257782, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9202858209609985, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.634102463722229, "metrics/particle/momentum/class_5/event/px_bias": 0.030502937734127045, "metrics/particle/momentum/class_5/event/py_bias": 0.07018090784549713, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5548268556594849, "metrics/particle/momentum/class_5/event/pz_bias": -0.2095808982849121, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.340918779373169, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2688641548156738, "step_valid_time": 5268.38254570961, "step_total_time": 5269.651410341263, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026546910472542, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06406237591339736, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2879, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2870, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548922056384743, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9507926023778072, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9528379943736555, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04920739762219287, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9519071310116086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9478203434610304, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9498593413867285, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548922056384743, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009694043885582, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07737105896327823, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2128, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2123, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9445184198845983, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.948306595365419, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9464087169223927, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05169340463458111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9422991566799822, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.946078431372549, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9441850122303758, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9445184198845983, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004429574597388, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07663239561336344, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1169, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9151750972762646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.92018779342723, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9176746000780336, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07981220657276995, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.909727626459144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9147104851330203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9122122512680453, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9151750972762646} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_32000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..ffec753092e115eed438bf9cdfdfdb7981d40675 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.5160128861095319, "train": {"Classification_binary": 2.468130350112915, "Classification": 0.020368466153740883, "Regression_pt": 0.13628855347633362, "Regression_eta": 0.008749261498451233, "Regression_sin_phi": 0.008704610168933868, "Regression_cos_phi": 0.010937502607703209, "Regression_energy": 0.13494226336479187, "Total": 8.620194435119629, "Total_unweighted": 2.7881210073828697}, "train_group": {}, "valid": {"Classification_binary": 2.549920399983724, "Classification": 0.020347898205121357, "Regression_pt": 0.13295460740725198, "Regression_eta": 0.010695607711871466, "Regression_sin_phi": 0.008132603019475937, "Regression_cos_phi": 0.008313645919164022, "Regression_energy": 0.13546433051427206, "Total": 8.553131739298502, "Total_unweighted": 2.865829092760881, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021049590781331062, "diagnostic/loss/clic_pf/Classification_binary": 2.563347578048706, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007941287942230701, "diagnostic/loss/clic_pf/Regression_energy": 0.1383243352174759, "diagnostic/loss/clic_pf/Regression_eta": 0.010082082822918892, "diagnostic/loss/clic_pf/Regression_pt": 0.13585805892944336, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007839269936084747, "diagnostic/regression/clic_pf/energy_pred_mean": 0.038187142461538315, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18854285776615143, "diagnostic/regression/clic_pf/energy_residual_mean": 0.053740762174129486, "diagnostic/regression/clic_pf/energy_residual_rms": 0.427301287651062, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.008612594567239285, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18490438163280487, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06023194640874863, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43154147267341614, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9649949073791504, "metrics/particle/class_1/pid_accuracy": 0.9874346256256104, "metrics/particle/class_2/efficiency": 0.4333558976650238, "metrics/particle/class_2/pid_accuracy": 0.756440281867981, "metrics/particle/class_3/efficiency": 0.750705897808075, "metrics/particle/class_3/pid_accuracy": 0.9852421879768372, "metrics/particle/class_4/efficiency": 0.9253876209259033, "metrics/particle/class_4/pid_accuracy": 0.39476439356803894, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.6991869807243347, "metrics/particle/count/bias_mean": -0.3686274588108063, "metrics/particle/count/mae": 3.1790850162506104, "metrics/particle/count/prediction_mean": 69.63072204589844, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.050041645765304565, "metrics/particle/event/energy_response_mean": 1.0017457008361816, "metrics/particle/event/met_abs_error": 4.502987384796143, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038279108703136444, "metrics/particle/event/scalar_pt_response_mean": 1.0007648468017578, "metrics/particle/event/vector_pt_closure": 0.0403645783662796, "metrics/particle/matched/delta_eta_abs_mean": 0.009112494997680187, "metrics/particle/matched/delta_phi_abs_mean": 0.009761590510606766, "metrics/particle/matched/delta_r_mean": 0.015024412423372269, "metrics/particle/matched/energy_relative_abs_mean": 0.08247449994087219, "metrics/particle/matched/pid_accuracy": 0.9685783386230469, "metrics/particle/matched/pt_relative_abs_mean": 0.07485788315534592, "metrics/particle/matching/duplicate_fraction": 0.02839442528784275, "metrics/particle/matching/efficiency": 0.8278788924217224, "metrics/particle/matching/f1": 0.8300644755363464, "metrics/particle/matching/purity": 0.8322616815567017, "metrics/particle/matching/target_coverage_dr0p05": 0.8029580116271973, "metrics/particle/matching/target_coverage_dr0p10": 0.8597652912139893, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7368602752685547, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8669175505638123, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017583107575774193, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08055401593446732, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07536039501428604, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0019843068439513445, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1314956694841385, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030186926014721394, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.33222803473472595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.01963958516716957, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2070759534835815, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8613.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28731176257133484, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9087483882904053, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9133945107460022, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7847193479537964, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1221008226275444, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.622283935546875, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.0133218327537179, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7274982333183289, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0885833129286766, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6089229583740234, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10434.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8012838363647461, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.7044565677642822, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.412046909332275, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.74784517288208, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.046531058847904205, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.16405484080314636, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.554136276245117, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.3050991296768188, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.172532081604004, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02694978378713131, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0913451537489891, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08957081288099289, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0048285298980772495, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1440265029668808, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021264994516968727, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9383319020271301, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.037320494651794434, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1560721397399902, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1834.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.744766891002655, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2285696268081665, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10413.0615234375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10412.8857421875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24982602894306183, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 558101.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07515657693147659, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8447818756103516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.14105243980884552, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.955596923828125, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2874.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5970051884651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0689685344696045, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.163654327392578, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0994797945022583, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.10543078184127808, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.030014336109161377, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.023534774780273, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.5666015148162842, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.247114181518555, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0129091152921319, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08277630060911179, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0735122337937355, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.012166401371359825, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1297420859336853, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008877748041413724, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11097129434347153, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010412395931780338, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5811814665794373, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19149.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14226596057415009, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5858559608459473, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.444083034992218, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3431437313556671, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08763722330331802, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.960841178894043, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002871832810342312, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24049142003059387, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.017176004126667976, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3888574838638306, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21589.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8722328543663025, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5469251871109009, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.395512104034424, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.264866352081299, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.011887334287166595, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7816517353057861, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1565823554992676, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.8110727071762085, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.270113945007324, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015139404684305191, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08266056329011917, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07504519820213318, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.008150079287588596, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1311817765235901, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00277064461261034, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22663095593452454, "metrics/particle/momentum/all/accepted/pz_bias": 0.01013978011906147, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1224150657653809, "metrics/particle/momentum/all/accepted/samples": 29596.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23525375127792358, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7353306412696838, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 858.1326904296875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 858.0173950195312, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11129896342754364, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 160163.125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011949451640248299, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6005842685699463, "metrics/particle/momentum/all/assigned/pz_bias": 0.02549520879983902, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1358654499053955, "metrics/particle/momentum/all/assigned/samples": 34897.0, "metrics/particle/momentum/all/efficiency": 0.8272815942764282, "metrics/particle/momentum/all/event/energy_bias": -2.2265000343322754, "metrics/particle/momentum/all/event/p3_error_mean": 13.97550106048584, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.54949426651001, "metrics/particle/momentum/all/event/px_bias": 0.07078705728054047, "metrics/particle/momentum/all/event/py_bias": -0.5875825881958008, "metrics/particle/momentum/all/event/pz_abs_mean": 10.723849296569824, "metrics/particle/momentum/all/event/pz_bias": 1.5495703220367432, "metrics/particle/momentum/all/event/pz_residual_rms": 18.646167755126953, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007407719735056162, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.034757066518068314, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021857397630810738, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0038691682275384665, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06268548220396042, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0018468357156962156, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11689502745866776, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.003448920324444771, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8440943956375122, "metrics/particle/momentum/class_1/accepted/samples": 16244.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04632056504487991, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.14075955748558044, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14050690829753876, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10988868027925491, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.024180643260478973, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.6986963748931885, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004741302691400051, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24774284660816193, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.027026358991861343, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4288649559020996, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9627215266227722, "metrics/particle/momentum/class_1/event/energy_bias": 3.113870620727539, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.555583953857422, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.129096031188965, "metrics/particle/momentum/class_1/event/px_bias": -0.4350668489933014, "metrics/particle/momentum/class_1/event/py_bias": -0.7355698347091675, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.030345916748047, "metrics/particle/momentum/class_1/event/pz_bias": 0.6790745258331299, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.488141059875488, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03623497858643532, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19609123468399048, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1970919966697693, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04244622960686684, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2350853532552719, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02911093644797802, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6144723892211914, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.02710619755089283, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.490333080291748, "metrics/particle/momentum/class_2/accepted/samples": 1271.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9130194187164307, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6617286801338196, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11185.677734375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11185.177734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36485981941223145, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 578488.4375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07626168429851532, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0884041786193848, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.041375961154699326, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.315493106842041, "metrics/particle/momentum/class_2/assigned/samples": 2675.0, "metrics/particle/momentum/class_2/efficiency": 0.43143245577812195, "metrics/particle/momentum/class_2/event/energy_bias": -9.130453109741211, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.257031440734863, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.678542613983154, "metrics/particle/momentum/class_2/event/px_bias": 0.48086103796958923, "metrics/particle/momentum/class_2/event/py_bias": -0.5952386260032654, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.88569450378418, "metrics/particle/momentum/class_2/event/pz_bias": -0.08552077412605286, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.352027893066406, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023941099643707275, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1377044916152954, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13790854811668396, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02002311684191227, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17928847670555115, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012970168609172106, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2284611165523529, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.02335353195667267, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9247514009475708, "metrics/particle/momentum/class_3/accepted/samples": 11565.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33411699533462524, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3953053951263428, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4550268650054932, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3101208209991455, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16710570454597473, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.162764549255371, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008848361670970917, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5596611499786377, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.027763353660702705, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.55615496635437, "metrics/particle/momentum/class_3/assigned/samples": 14805.0, "metrics/particle/momentum/class_3/efficiency": 0.7503892779350281, "metrics/particle/momentum/class_3/event/energy_bias": 4.585795879364014, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.448226928710938, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.968808174133301, "metrics/particle/momentum/class_3/event/px_bias": -0.12327418476343155, "metrics/particle/momentum/class_3/event/py_bias": 0.585412323474884, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.988498687744141, "metrics/particle/momentum/class_3/event/pz_bias": 0.8170073628425598, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.929945945739746, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011396928690373898, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10755646973848343, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05215378478169441, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0015101772733032703, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09536144882440567, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23944254219532013, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0566929429769516, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8660879731178284, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09519923478364944, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.352210521697998, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7937568426132202, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.7391419410705566, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.960243225097656, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3684779107570648, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.03586483746767044, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2804155349731445, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -0.4764326810836792, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8021256923675537, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1728606224060059, "metrics/particle/momentum/class_4/event/px_bias": 0.12023768573999405, "metrics/particle/momentum/class_4/event/py_bias": 0.055904850363731384, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1565531492233276, "metrics/particle/momentum/class_4/event/pz_bias": 0.33111169934272766, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.286625385284424, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00533254211768508, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02072967402637005, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.016528036445379257, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0012940751621499658, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0481068454682827, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1879117488861084, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06023109704256058, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6883708834648132, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.025784745812416077, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03669252246618271, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03246240317821503, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006913028657436371, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12251640856266022, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5372013449668884, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4123269021511078, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.153550624847412, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.319280743598938, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.00855553150177, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7046420574188232, "metrics/particle/momentum/class_5/event/px_bias": 0.02802933007478714, "metrics/particle/momentum/class_5/event/py_bias": 0.10190878808498383, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6003867983818054, "metrics/particle/momentum/class_5/event/pz_bias": -0.19210249185562134, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.401963472366333, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3704595565795898, "step_valid_time": 5408.621750831604, "step_total_time": 5409.992210865021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000642181540842, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06221495701219326, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2890, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585406301824212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9563203176704169, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9574291866821268, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04367968232958306, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9562189054726368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540039708802118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551101540500249, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585406301824212, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9969123338891817, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07267847914403569, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2124, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2121, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9427430093209055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473684210526315, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9450500556173527, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05263157894736842, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9414114513981359, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9460303300624442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9437152391546162, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9427430093209055, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000219481914125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0845145016351937, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1275, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9231372549019607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.91953125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07686274509803921, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91284046692607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.92, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.91640625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9159533073929961} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_33000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..07110a1175909ed8bafb301edb93cbf74b074ae1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5633882888004549, "train": {"Classification_binary": 2.568182945251465, "Classification": 0.019733037799596786, "Regression_pt": 0.13789376616477966, "Regression_eta": 0.009476139210164547, "Regression_sin_phi": 0.007585268467664719, "Regression_cos_phi": 0.009982969611883163, "Regression_energy": 0.13520681858062744, "Total": 8.615652084350586, "Total_unweighted": 2.888060945086181}, "train_group": {}, "valid": {"Classification_binary": 2.5363232294718423, "Classification": 0.020107171187798183, "Regression_pt": 0.1326043407122294, "Regression_eta": 0.010640863329172134, "Regression_sin_phi": 0.008075226098299026, "Regression_cos_phi": 0.008303731059034666, "Regression_energy": 0.13498249650001526, "Total": 8.51237678527832, "Total_unweighted": 2.851037058358391, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020780224353075027, "diagnostic/loss/clic_pf/Classification_binary": 2.549485445022583, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007939045317471027, "diagnostic/loss/clic_pf/Regression_energy": 0.13724330067634583, "diagnostic/loss/clic_pf/Regression_eta": 0.009999632835388184, "diagnostic/loss/clic_pf/Regression_pt": 0.1347566545009613, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007790359668433666, "diagnostic/regression/clic_pf/energy_pred_mean": 0.05425774306058884, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19119004905223846, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0698113664984703, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4306626319885254, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.013077294453978539, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18901707231998444, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08192183077335358, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43650302290916443, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9660484790802002, "metrics/particle/class_1/pid_accuracy": 0.9867486953735352, "metrics/particle/class_2/efficiency": 0.4407983720302582, "metrics/particle/class_2/pid_accuracy": 0.7518547177314758, "metrics/particle/class_3/efficiency": 0.7445414662361145, "metrics/particle/class_3/pid_accuracy": 0.9851489663124084, "metrics/particle/class_4/efficiency": 0.9244186282157898, "metrics/particle/class_4/pid_accuracy": 0.4109014570713043, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7058823704719543, "metrics/particle/count/bias_mean": -0.3222222328186035, "metrics/particle/count/mae": 3.136601209640503, "metrics/particle/count/prediction_mean": 69.6771240234375, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05364057794213295, "metrics/particle/event/energy_response_mean": 1.023511528968811, "metrics/particle/event/met_abs_error": 4.744140148162842, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04397893697023392, "metrics/particle/event/scalar_pt_response_mean": 1.0247910022735596, "metrics/particle/event/vector_pt_closure": 0.041949618607759476, "metrics/particle/matched/delta_eta_abs_mean": 0.009333292953670025, "metrics/particle/matched/delta_phi_abs_mean": 0.01017539668828249, "metrics/particle/matched/delta_r_mean": 0.01549856923520565, "metrics/particle/matched/energy_relative_abs_mean": 0.08373990654945374, "metrics/particle/matched/pid_accuracy": 0.9679766893386841, "metrics/particle/matched/pt_relative_abs_mean": 0.077479287981987, "metrics/particle/matching/duplicate_fraction": 0.029791943728923798, "metrics/particle/matching/efficiency": 0.8263195753097534, "metrics/particle/matching/f1": 0.82822585105896, "metrics/particle/matching/purity": 0.8301408886909485, "metrics/particle/matching/target_coverage_dr0p05": 0.8063194155693054, "metrics/particle/matching/target_coverage_dr0p10": 0.8626877665519714, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7271309494972229, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8633320331573486, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018033910542726517, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08148179948329926, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07825694233179092, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.026346370577812195, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13536277413368225, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0025682933628559113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32757991552352905, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008865488693118095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1553257703781128, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8566.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28895193338394165, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0159786939620972, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9744672775268555, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8673750758171082, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1294061243534088, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.977914810180664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01293103490024805, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7406591176986694, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.038203395903110504, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6033685207366943, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10440.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.796911358833313, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.796205759048462, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.1960296630859375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7787985801696777, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.240886390209198, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.23204275965690613, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.327314376831055, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.44713571667671204, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.879191398620605, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02681897208094597, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09719454497098923, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09825900942087173, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.036891333758831024, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1563243865966797, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017070485278964043, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9474425315856934, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06149328127503395, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.896087408065796, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1816.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7806527018547058, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4155412912368774, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3695.418212890625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3695.275634765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.263874351978302, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 197613.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0726003497838974, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7981255054473877, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11992129683494568, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.36666202545166, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2865.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5911458134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.2651023268699646, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.461091995239258, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0593862533569336, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.029753923416137695, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11191066354513168, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.311857223510742, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.06996269524097443, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.181108474731445, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013463611714541912, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08379147201776505, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07540877163410187, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02104722335934639, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13282647728919983, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005749529809691012, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11087164282798767, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0027038983535021544, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5653354525566101, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19132.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14317584037780762, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6404879689216614, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4891839623451233, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.39704430103302, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08812633156776428, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.19819974899292, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026859312783926725, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2426673173904419, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.005841420032083988, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.367732048034668, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21594.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8714585304260254, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.905251979827881, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.981558322906494, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.99743127822876, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.29860997200012207, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.06190748140215874, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7800779342651367, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.08121936768293381, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.2137346267700195, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015611831098794937, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08394581824541092, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07764138281345367, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023560112342238426, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13512349128723145, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0021684623789042234, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22524231672286987, "metrics/particle/momentum/all/accepted/pz_bias": -0.00810952577739954, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0538742542266846, "metrics/particle/momentum/all/accepted/samples": 29514.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23911765217781067, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8164429664611816, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 303.96600341796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 303.86529541015625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11490300297737122, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 56620.4609375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011490300297737122, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6014289855957031, "metrics/particle/momentum/all/assigned/pz_bias": 0.017658909782767296, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.988588571548462, "metrics/particle/momentum/all/assigned/samples": 34899.0, "metrics/particle/momentum/all/efficiency": 0.8249894976615906, "metrics/particle/momentum/all/event/energy_bias": 4.4363555908203125, "metrics/particle/momentum/all/event/p3_error_mean": 13.503623962402344, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.5994977951049805, "metrics/particle/momentum/all/event/px_bias": -0.02796967327594757, "metrics/particle/momentum/all/event/py_bias": 0.40586090087890625, "metrics/particle/momentum/all/event/pz_abs_mean": 10.186576843261719, "metrics/particle/momentum/all/event/pz_bias": -0.295953631401062, "metrics/particle/momentum/all/event/pz_residual_rms": 17.422121047973633, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008036265149712563, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.034062597900629044, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021856257691979408, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0038576265797019005, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06227027252316475, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011677216971293092, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11325496435165405, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.003317247610539198, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7930160164833069, "metrics/particle/momentum/class_1/accepted/samples": 16271.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04817565903067589, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11921285837888718, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12820298969745636, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09780605137348175, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022522522136569023, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.312070846557617, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089615773409605, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23600372672080994, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.010173945687711239, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1220591068267822, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9643216729164124, "metrics/particle/momentum/class_1/event/energy_bias": 2.5092380046844482, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.218326568603516, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.016814708709717, "metrics/particle/momentum/class_1/event/px_bias": -0.7097275257110596, "metrics/particle/momentum/class_1/event/py_bias": -0.5446401238441467, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.697728633880615, "metrics/particle/momentum/class_1/event/pz_bias": 0.13835901021957397, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.395942687988281, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03647584840655327, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19671158492565155, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19704441726207733, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02166232094168663, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2358628511428833, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02625482715666294, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5889443159103394, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.007037779316306114, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.395139217376709, "metrics/particle/momentum/class_2/accepted/samples": 1295.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9297628402709961, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7552272081375122, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3928.819580078125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3928.346435546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3609357476234436, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 203827.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07389528304338455, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.061279058456421, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.19775576889514923, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.155733108520508, "metrics/particle/momentum/class_2/assigned/samples": 2693.0, "metrics/particle/momentum/class_2/efficiency": 0.4395790994167328, "metrics/particle/momentum/class_2/event/energy_bias": -5.290424346923828, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.077438354492188, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.724011421203613, "metrics/particle/momentum/class_2/event/px_bias": 0.20403483510017395, "metrics/particle/momentum/class_2/event/py_bias": -0.29930245876312256, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.635418891906738, "metrics/particle/momentum/class_2/event/pz_bias": -0.13794714212417603, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.86511516571045, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02429509349167347, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14228416979312897, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14496934413909912, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05750632658600807, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1868780553340912, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000961958896368742, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22833047807216644, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.015379255637526512, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.791816771030426, "metrics/particle/momentum/class_3/accepted/samples": 11435.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3371802270412445, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.608845591545105, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.695918083190918, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5806711912155151, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17802569270133972, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.66592025756836, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008857335895299911, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5759634375572205, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.001305332058109343, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.521820306777954, "metrics/particle/momentum/class_3/assigned/samples": 14790.0, "metrics/particle/momentum/class_3/efficiency": 0.7419543266296387, "metrics/particle/momentum/class_3/event/energy_bias": 7.930320739746094, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.027071952819824, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.38472843170166, "metrics/particle/momentum/class_3/event/px_bias": 0.28380003571510315, "metrics/particle/momentum/class_3/event/py_bias": 1.0575039386749268, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.352079391479492, "metrics/particle/momentum/class_3/event/pz_bias": -0.3877168893814087, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.467738151550293, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011426098644733429, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11057426780462265, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05718651041388512, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006304716691374779, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10384982824325562, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26622825860977173, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030448108911514282, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8962255120277405, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09997693449258804, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7850679159164429, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.5131144523620605, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.457634687423706, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.562061309814453, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.39325183629989624, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06459154188632965, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2627650499343872, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.4098847508430481, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8040879964828491, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1541926860809326, "metrics/particle/momentum/class_4/event/px_bias": 0.11948949098587036, "metrics/particle/momentum/class_4/event/py_bias": 0.07081469148397446, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1784636974334717, "metrics/particle/momentum/class_4/event/pz_bias": 0.2702464759349823, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.267179489135742, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0063832770101726055, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025986233726143837, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021717002615332603, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005295167211443186, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06647830456495285, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2651841938495636, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1608460396528244, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.0088598728179932, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.028624998405575752, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03755269572138786, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03324708715081215, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0063664671033620834, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1160673052072525, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5633682608604431, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4607499837875366, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.193150997161865, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.3028939962387085, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1047898530960083, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7776515483856201, "metrics/particle/momentum/class_5/event/px_bias": 0.07443352043628693, "metrics/particle/momentum/class_5/event/py_bias": 0.1214848980307579, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6513125896453857, "metrics/particle/momentum/class_5/event/pz_bias": -0.17889508605003357, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.449906349182129, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4725661277770996, "step_valid_time": 5549.523745536804, "step_total_time": 5550.9963121414185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0059986638968856, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06314900282319791, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9592039800995025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954140547674035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566655640092623, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04585945232596503, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952161002969317, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546807806814422, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9592039800995025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0125307721509824, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07941263571471513, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2131, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9502000889284127, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9493558418480675, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04979991107158737, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9458499778073679, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9475322365495776, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9466903598400711, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9485130936529073, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0166113901339981, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08412942000394041, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1179, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221789883268483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9207459207459208, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.921461897356143, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07925407925407925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9175097276264591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.916083916083916, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.916796267496112, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221789883268483} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_34000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0a55c66987381ade21c5cecf3da5236853188c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.6107636914913777, "train": {"Classification_binary": 2.52433180809021, "Classification": 0.019986940547823906, "Regression_pt": 0.1421169638633728, "Regression_eta": 0.011508719064295292, "Regression_sin_phi": 0.008201097138226032, "Regression_cos_phi": 0.008889345452189445, "Regression_energy": 0.14423921704292297, "Total": 8.838231086730957, "Total_unweighted": 2.8592740911990404}, "train_group": {}, "valid": {"Classification_binary": 2.5296414693196616, "Classification": 0.020075971881548565, "Regression_pt": 0.13206892212231955, "Regression_eta": 0.010887227952480316, "Regression_sin_phi": 0.008118707686662674, "Regression_cos_phi": 0.008326833446820578, "Regression_energy": 0.13483959436416626, "Total": 8.509905497233072, "Total_unweighted": 2.8439587267736597, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020598599687218666, "diagnostic/loss/clic_pf/Classification_binary": 2.5387346744537354, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007961271330714226, "diagnostic/loss/clic_pf/Regression_energy": 0.13733388483524323, "diagnostic/loss/clic_pf/Regression_eta": 0.010171439498662949, "diagnostic/loss/clic_pf/Regression_pt": 0.13491937518119812, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007888994179666042, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03791571035981178, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18595373630523682, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05346933379769325, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42024099826812744, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.004421695601195097, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18400682508945465, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0644228458404541, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42720919847488403, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9664658904075623, "metrics/particle/class_1/pid_accuracy": 0.9864459037780762, "metrics/particle/class_2/efficiency": 0.44632387161254883, "metrics/particle/class_2/pid_accuracy": 0.7602324485778809, "metrics/particle/class_3/efficiency": 0.7527966499328613, "metrics/particle/class_3/pid_accuracy": 0.984538733959198, "metrics/particle/class_4/efficiency": 0.9234496355056763, "metrics/particle/class_4/pid_accuracy": 0.4218258261680603, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7342799305915833, "metrics/particle/count/bias_mean": -0.3633987009525299, "metrics/particle/count/mae": 3.0601308345794678, "metrics/particle/count/prediction_mean": 69.63594818115234, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04905407130718231, "metrics/particle/event/energy_response_mean": 1.0104299783706665, "metrics/particle/event/met_abs_error": 4.762971878051758, "metrics/particle/event/scalar_pt_relative_abs_error": 0.039138831198215485, "metrics/particle/event/scalar_pt_response_mean": 1.0106583833694458, "metrics/particle/event/vector_pt_closure": 0.04102892056107521, "metrics/particle/matched/delta_eta_abs_mean": 0.009954816661775112, "metrics/particle/matched/delta_phi_abs_mean": 0.010038566775619984, "metrics/particle/matched/delta_r_mean": 0.015901779755949974, "metrics/particle/matched/energy_relative_abs_mean": 0.08185912668704987, "metrics/particle/matched/pid_accuracy": 0.9681843519210815, "metrics/particle/matched/pt_relative_abs_mean": 0.07505235821008682, "metrics/particle/matching/duplicate_fraction": 0.029246406629681587, "metrics/particle/matching/efficiency": 0.8305399417877197, "metrics/particle/matching/f1": 0.8327014446258545, "metrics/particle/matching/purity": 0.834874153137207, "metrics/particle/matching/target_coverage_dr0p05": 0.8045079708099365, "metrics/particle/matching/target_coverage_dr0p10": 0.8627811670303345, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7377379536628723, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8677672147750854, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017660973593592644, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07821369916200638, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07357413321733475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009570647962391376, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12889917194843292, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0032584662549197674, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3240029811859131, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02607463300228119, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2206734418869019, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8593.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29270505905151367, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.850380539894104, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8767319917678833, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7558691501617432, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12569564580917358, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.349898338317871, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013049318455159664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7164127826690674, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.049611467868089676, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.51627779006958, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10422.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7994232177734375, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8083105683326721, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.292871475219727, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8493900299072266, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5599769949913025, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3965967297554016, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.401026725769043, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2542628347873688, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.046526908874512, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026597222313284874, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09306425601243973, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08886831998825073, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.014534842222929, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14643895626068115, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020982882007956505, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.970748245716095, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.08053971081972122, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.7431039810180664, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1811.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7710456848144531, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.428529143333435, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3515.45458984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3515.2900390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2632681429386139, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 188009.859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07891061156988144, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1544315814971924, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07637795060873032, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.364253044128418, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2864.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5895182490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1963355541229248, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.664440631866455, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.03921377658844, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11306728422641754, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0934293121099472, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.526578903198242, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.28455018997192383, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.142553329467773, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014127785339951515, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0821673646569252, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07405944168567657, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014147666282951832, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13037265837192535, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006757810479030013, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11319141834974289, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.022326555103063583, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6043049693107605, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19237.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13944604992866516, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7042887210845947, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5621364116668701, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46509820222854614, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08297105133533478, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 10.286375999450684, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027287022676318884, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2417869120836258, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0077496725134551525, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3540964126586914, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21622.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8762412071228027, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.8887295722961426, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.526818752288818, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.330472946166992, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.02735447883605957, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.5077677965164185, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.2364673614501953, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.24504491686820984, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.768542289733887, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01591392047703266, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08168695867061615, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0748235359787941, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012844432145357132, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13098855316638947, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026652272790670395, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2267010658979416, "metrics/particle/momentum/all/accepted/pz_bias": 0.026969829574227333, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.234910011291504, "metrics/particle/momentum/all/accepted/samples": 29641.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2370215803384781, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8073250651359558, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 289.03271484375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 288.9230651855469, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11051908135414124, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 53852.3515625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012060272507369518, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6224551200866699, "metrics/particle/momentum/all/assigned/pz_bias": 0.02587832510471344, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.6907689571380615, "metrics/particle/momentum/all/assigned/samples": 34908.0, "metrics/particle/momentum/all/efficiency": 0.8285394906997681, "metrics/particle/momentum/all/event/energy_bias": -0.1159164309501648, "metrics/particle/momentum/all/event/p3_error_mean": 13.620868682861328, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.633999824523926, "metrics/particle/momentum/all/event/px_bias": 0.7003986835479736, "metrics/particle/momentum/all/event/py_bias": 0.9977939128875732, "metrics/particle/momentum/all/event/pz_abs_mean": 10.329343795776367, "metrics/particle/momentum/all/event/pz_bias": 0.7838579416275024, "metrics/particle/momentum/all/event/pz_residual_rms": 17.97618293762207, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007521564606577158, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03296537324786186, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021446308121085167, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0025289335753768682, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06161869317293167, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014759240439161658, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10906017571687698, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.009408554062247276, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8283106088638306, "metrics/particle/momentum/class_1/accepted/samples": 16261.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04798758774995804, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09454600512981415, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08476122468709946, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05243303254246712, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021753303706645966, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6680824756622314, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0047418647445738316, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.24150051176548004, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.018928460776805878, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3395605087280273, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9637290239334106, "metrics/particle/momentum/class_1/event/energy_bias": 2.659998893737793, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.250131607055664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.799553871154785, "metrics/particle/momentum/class_1/event/px_bias": -0.40178096294403076, "metrics/particle/momentum/class_1/event/py_bias": 0.02768785133957863, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.945837020874023, "metrics/particle/momentum/class_1/event/pz_bias": 0.42024916410446167, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.295373916625977, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03943600878119469, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19393537938594818, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19266542792320251, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0477980338037014, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2317321002483368, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03211009129881859, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6117985248565674, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.15409965813159943, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.521622657775879, "metrics/particle/momentum/class_2/accepted/samples": 1308.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9033915400505066, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6889634132385254, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3745.391845703125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3744.910400390625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35653144121170044, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 194103.46875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07778191566467285, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.127803325653076, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10887906700372696, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.160387992858887, "metrics/particle/momentum/class_2/assigned/samples": 2687.0, "metrics/particle/momentum/class_2/efficiency": 0.44399183988571167, "metrics/particle/momentum/class_2/event/energy_bias": -7.97080659866333, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.42153549194336, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.854671478271484, "metrics/particle/momentum/class_2/event/px_bias": 0.17490556836128235, "metrics/particle/momentum/class_2/event/py_bias": 0.31480199098587036, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.895317077636719, "metrics/particle/momentum/class_2/event/pz_bias": 0.02414563298225403, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.1527042388916, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025338584557175636, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13786569237709045, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13802562654018402, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03465747460722923, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17958416044712067, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001124859438277781, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23535028100013733, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.036698438227176666, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2307088375091553, "metrics/particle/momentum/class_3/accepted/samples": 11557.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33765730261802673, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6224671602249146, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5917036533355713, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4591431617736816, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16939078271389008, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.924560546875, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008780224248766899, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6088612675666809, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.023886214941740036, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.680307388305664, "metrics/particle/momentum/class_3/assigned/samples": 14806.0, "metrics/particle/momentum/class_3/efficiency": 0.749870240688324, "metrics/particle/momentum/class_3/event/energy_bias": 6.042229652404785, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.779340744018555, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.2467451095581055, "metrics/particle/momentum/class_3/event/px_bias": 0.7272847890853882, "metrics/particle/momentum/class_3/event/py_bias": 0.5176337957382202, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.239799976348877, "metrics/particle/momentum/class_3/event/pz_bias": 0.2847643494606018, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.365560531616211, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011668621562421322, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10324563086032867, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.057122159749269485, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004622896201908588, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1030440405011177, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.261004239320755, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06877942383289337, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8742123246192932, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09385239332914352, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.9531418085098267, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8877789974212646, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.824542760848999, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.08199691772461, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3903827667236328, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.010892214253544807, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.26423978805542, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -0.5922482013702393, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7690181732177734, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1334948539733887, "metrics/particle/momentum/class_4/event/px_bias": 0.10257188230752945, "metrics/particle/momentum/class_4/event/py_bias": 0.04611453786492348, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1627784967422485, "metrics/particle/momentum/class_4/event/pz_bias": 0.17946697771549225, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.115469455718994, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005863640923053026, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019429152831435204, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015099047683179379, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00013653138012159616, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04127076640725136, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16901665925979614, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01373414508998394, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5905486941337585, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019830873236060143, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.035647518932819366, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03136832267045975, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.005934750195592642, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.121161550283432, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.515285849571228, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3634161353111267, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1346940994262695, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.25508975982666016, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9344685673713684, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6315304040908813, "metrics/particle/momentum/class_5/event/px_bias": 0.09741751104593277, "metrics/particle/momentum/class_5/event/py_bias": 0.09155572205781937, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5798935890197754, "metrics/particle/momentum/class_5/event/pz_bias": -0.12476824223995209, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4138023853302, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4471287727355957, "step_valid_time": 5690.134434223175, "step_total_time": 5691.581563234329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013223453166533, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06332144005804918, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2889, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2881, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582089552238806, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9563058589870904, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9572564612326043, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04369414101290963, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9555555555555556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536577292287322, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546056991385024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582089552238806, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002099258927091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07675370231273371, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9471815357301376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526785714285714, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9499221010460717, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04732142857142857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467376830892144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522321428571429, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494769641664812, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9471815357301376, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0055910907134782, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07751222581583417, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1189, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9252918287937744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.92890625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9270955165692008, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07109375, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9221789883268483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.92578125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9239766081871345, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9252918287937744} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_35000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..69c59ba999fe5f29e557db22ca7523f9e502a983 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6581390941823007, "train": {"Classification_binary": 2.425107002258301, "Classification": 0.020090021193027496, "Regression_pt": 0.13339866697788239, "Regression_eta": 0.008783433586359024, "Regression_sin_phi": 0.008232010528445244, "Regression_cos_phi": 0.008110162802040577, "Regression_energy": 0.1321306675672531, "Total": 8.297837257385254, "Total_unweighted": 2.7358519649133086}, "train_group": {}, "valid": {"Classification_binary": 2.5236258506774902, "Classification": 0.02004145085811615, "Regression_pt": 0.13053247332572937, "Regression_eta": 0.010568946599960327, "Regression_sin_phi": 0.008028306066989899, "Regression_cos_phi": 0.008297995974620184, "Regression_energy": 0.13314821322758993, "Total": 8.435413360595703, "Total_unweighted": 2.834243236730496, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020744837820529938, "diagnostic/loss/clic_pf/Classification_binary": 2.535081386566162, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007931118831038475, "diagnostic/loss/clic_pf/Regression_energy": 0.13655385375022888, "diagnostic/loss/clic_pf/Regression_eta": 0.00985793862491846, "diagnostic/loss/clic_pf/Regression_pt": 0.13415761291980743, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007737529464066029, "diagnostic/regression/clic_pf/energy_pred_mean": 0.044798046350479126, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18811751902103424, "diagnostic/regression/clic_pf/energy_residual_mean": 0.060351673513650894, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4272887110710144, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0032456503249704838, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.186372771859169, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07209018617868423, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43340760469436646, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9675790667533875, "metrics/particle/class_1/pid_accuracy": 0.9865642189979553, "metrics/particle/class_2/efficiency": 0.44181326031684875, "metrics/particle/class_2/pid_accuracy": 0.7611026167869568, "metrics/particle/class_3/efficiency": 0.7420195937156677, "metrics/particle/class_3/pid_accuracy": 0.9855632185935974, "metrics/particle/class_4/efficiency": 0.9273256063461304, "metrics/particle/class_4/pid_accuracy": 0.42424243688583374, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7154471278190613, "metrics/particle/count/bias_mean": -0.9862744808197021, "metrics/particle/count/mae": 3.2790849208831787, "metrics/particle/count/prediction_mean": 69.01306915283203, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04974615201354027, "metrics/particle/event/energy_response_mean": 1.0100842714309692, "metrics/particle/event/met_abs_error": 4.486688137054443, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03958938643336296, "metrics/particle/event/scalar_pt_response_mean": 1.010817289352417, "metrics/particle/event/vector_pt_closure": 0.041335783898830414, "metrics/particle/matched/delta_eta_abs_mean": 0.007928257808089256, "metrics/particle/matched/delta_phi_abs_mean": 0.010096901096403599, "metrics/particle/matched/delta_r_mean": 0.014366736635565758, "metrics/particle/matched/energy_relative_abs_mean": 0.08089835941791534, "metrics/particle/matched/pid_accuracy": 0.9685991406440735, "metrics/particle/matched/pt_relative_abs_mean": 0.0749516561627388, "metrics/particle/matching/duplicate_fraction": 0.02908419445157051, "metrics/particle/matching/efficiency": 0.8260487914085388, "metrics/particle/matching/f1": 0.8319094777107239, "metrics/particle/matching/purity": 0.8378539681434631, "metrics/particle/matching/target_coverage_dr0p05": 0.8045453429222107, "metrics/particle/matching/target_coverage_dr0p10": 0.8589622974395752, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7355064153671265, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8630519509315491, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016839131712913513, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0787920206785202, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07446879893541336, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010342924855649471, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13013264536857605, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0032698819413781166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32800793647766113, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.015169912949204445, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2243146896362305, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8563.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2838344871997833, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9093613028526306, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.053132176399231, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9332565069198608, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12349368631839752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.600749969482422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01368938572704792, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7378866076469421, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.020288387313485146, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.637848377227783, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10373.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7966322302818298, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0073325634002686, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.367000579833984, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8884634971618652, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.2748143672943115, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.27065473794937134, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.421032428741455, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.17686015367507935, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.34824275970459, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02551320381462574, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09431031346321106, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09134837239980698, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01668407954275608, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14899979531764984, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.024242423474788666, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0274263620376587, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0401080958545208, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4177069664001465, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1815.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7289543151855469, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1876028776168823, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.77408766746521, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.623105764389038, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2451154589653015, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.19465255737305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07460035383701324, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7877209186553955, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07217596471309662, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.643179893493652, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2815.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5908203125, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.416633665561676, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.008519172668457, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1514402627944946, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09847790002822876, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.04768862575292587, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.844362258911133, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.8458424806594849, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.26342010498047, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01231768261641264, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08085890859365463, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07385066151618958, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01657252013683319, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13049837946891785, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005224114283919334, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10926814377307892, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005087706726044416, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5903362035751343, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19142.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1348257213830948, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.707584023475647, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.578127384185791, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4852201044559479, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08656522631645203, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.830744743347168, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025978845078498125, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2446984052658081, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0013754368992522359, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8828917741775513, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21556.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8719140291213989, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.02052903175354, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.220346450805664, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.17685604095459, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6245201826095581, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5858925580978394, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.983008623123169, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4218162000179291, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.042784690856934, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014440549537539482, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0810864046216011, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07510579377412796, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014772331342101097, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13160590827465057, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027777778450399637, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2291707992553711, "metrics/particle/momentum/all/accepted/pz_bias": -0.010165482759475708, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1742914915084839, "metrics/particle/momentum/all/accepted/samples": 29520.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22745005786418915, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8067173957824707, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8978618383407593, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7921977043151855, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1104363352060318, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.22490882873535, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011743035167455673, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5979809761047363, "metrics/particle/momentum/all/assigned/pz_bias": -0.0006439323769882321, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2139768600463867, "metrics/particle/momentum/all/assigned/samples": 34744.0, "metrics/particle/momentum/all/efficiency": 0.8251572251319885, "metrics/particle/momentum/all/event/energy_bias": 0.5965627431869507, "metrics/particle/momentum/all/event/p3_error_mean": 13.914016723632812, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.722112655639648, "metrics/particle/momentum/all/event/px_bias": -0.8008566498756409, "metrics/particle/momentum/all/event/py_bias": -0.26754915714263916, "metrics/particle/momentum/all/event/pz_abs_mean": 10.538708686828613, "metrics/particle/momentum/all/event/pz_bias": -0.6008864045143127, "metrics/particle/momentum/all/event/pz_residual_rms": 19.915788650512695, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007420898415148258, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.032007284462451935, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022271640598773956, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005288842599838972, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06310752779245377, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015965612838044763, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11582673341035843, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012718124315142632, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8663906455039978, "metrics/particle/momentum/class_1/accepted/samples": 16285.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04468320310115814, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10908274352550507, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10574611276388168, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07662753015756607, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02163347601890564, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.247870445251465, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004623044282197952, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23816651105880737, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0006122991326265037, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.270946979522705, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9651514291763306, "metrics/particle/momentum/class_1/event/energy_bias": 2.709362268447876, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.477325439453125, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.082743167877197, "metrics/particle/momentum/class_1/event/px_bias": -0.8393129110336304, "metrics/particle/momentum/class_1/event/py_bias": -0.27768537402153015, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.970834732055664, "metrics/particle/momentum/class_1/event/pz_bias": 0.2167576104402542, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.506118774414062, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03653806075453758, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19873124361038208, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1975463330745697, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.041499100625514984, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.236455500125885, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03384615480899811, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6710339784622192, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0197380892932415, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.508186101913452, "metrics/particle/momentum/class_2/accepted/samples": 1300.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8865535855293274, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8411350846290588, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.0220271348953247, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5435819625854492, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35113635659217834, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7.077737808227539, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07727272808551788, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0900936126708984, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13484326004981995, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.104851722717285, "metrics/particle/momentum/class_2/assigned/samples": 2640.0, "metrics/particle/momentum/class_2/efficiency": 0.44127631187438965, "metrics/particle/momentum/class_2/event/energy_bias": -7.471378326416016, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.179686546325684, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.675610065460205, "metrics/particle/momentum/class_2/event/px_bias": 0.027738533914089203, "metrics/particle/momentum/class_2/event/py_bias": -0.9278066158294678, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.796134948730469, "metrics/particle/momentum/class_2/event/pz_bias": 0.0350915789604187, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.07944107055664, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02216903120279312, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1378476619720459, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13794934749603271, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03495585545897484, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17997674643993378, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010507880942896008, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22636859118938446, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0047864755615592, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0304137468338013, "metrics/particle/momentum/class_3/accepted/samples": 11420.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3248624801635742, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5768332481384277, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7651816606521606, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6365429162979126, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1715686321258545, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.627490997314453, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008442265912890434, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5692065358161926, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.019343221560120583, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5382938385009766, "metrics/particle/momentum/class_3/assigned/samples": 14688.0, "metrics/particle/momentum/class_3/efficiency": 0.7409810423851013, "metrics/particle/momentum/class_3/event/energy_bias": 6.167515277862549, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.617972373962402, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.019839286804199, "metrics/particle/momentum/class_3/event/px_bias": -0.19526007771492004, "metrics/particle/momentum/class_3/event/py_bias": 0.8370790481567383, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.09459924697876, "metrics/particle/momentum/class_3/event/pz_bias": -0.822963535785675, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.325371742248535, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011106458492577076, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10740552097558975, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0566534660756588, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.011585559695959091, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10023051500320435, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2558777928352356, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.010746694169938564, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8203535676002502, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08446399867534637, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.211289644241333, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1536853313446045, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0963432788848877, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 35.03505325317383, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3460502028465271, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02649124339222908, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0230573415756226, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -0.4239584803581238, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.760351538658142, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.120849370956421, "metrics/particle/momentum/class_4/event/px_bias": 0.09725600481033325, "metrics/particle/momentum/class_4/event/py_bias": 0.05995678901672363, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1461913585662842, "metrics/particle/momentum/class_4/event/pz_bias": 0.24094942212104797, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.352369785308838, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0057471077889204025, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020592112094163895, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017671402543783188, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004489313345402479, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05035140737891197, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19763512909412384, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09140989929437637, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8587738871574402, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.032813675701618195, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.035898562520742416, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03510691970586777, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0012018870329484344, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13501477241516113, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6279228925704956, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5219043493270874, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.354051113128662, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3849785327911377, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9537711143493652, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6856470704078674, "metrics/particle/momentum/class_5/event/px_bias": 0.1087217926979065, "metrics/particle/momentum/class_5/event/py_bias": 0.04090705141425133, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5399332642555237, "metrics/particle/momentum/class_5/event/pz_bias": -0.2707214653491974, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2754013538360596, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3986737728118896, "step_valid_time": 5831.238473415375, "step_total_time": 5832.637147665024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020034405151546, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06159666514539597, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2882, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558872305140962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9533575917962289, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9546207353428288, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04664240820377109, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535655058043118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9510420112471055, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9523020867837032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558872305140962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026649744745695, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07440852966964573, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9538393253439857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538393253439857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9538393253439857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0461606746560142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.952063914780293, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952063914780293, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.952063914780293, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9538393253439857, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0039129728048266, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07728098479286771, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9242779078844653, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9228371005455963, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07572209211553474, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9151750972762646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9180327868852459, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.916601714731099, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9214007782101167} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_36000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..df01254dbc18f1a1dd5d1802b52554a57e42e4ae --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.7055144968732234, "train": {"Classification_binary": 2.422607183456421, "Classification": 0.01936337538063526, "Regression_pt": 0.12715688347816467, "Regression_eta": 0.011521532200276852, "Regression_sin_phi": 0.0070755756460130215, "Regression_cos_phi": 0.006758745759725571, "Regression_energy": 0.1285250335931778, "Total": 8.080551147460938, "Total_unweighted": 2.723008329514414}, "train_group": {}, "valid": {"Classification_binary": 2.494497617085775, "Classification": 0.01980942239363988, "Regression_pt": 0.129401425520579, "Regression_eta": 0.010584767907857895, "Regression_sin_phi": 0.008086757734417915, "Regression_cos_phi": 0.008250622699658075, "Regression_energy": 0.13199763496716818, "Total": 8.369089126586914, "Total_unweighted": 2.802628248309096, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02046441286802292, "diagnostic/loss/clic_pf/Classification_binary": 2.497108221054077, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007869245484471321, "diagnostic/loss/clic_pf/Regression_energy": 0.1345994472503662, "diagnostic/loss/clic_pf/Regression_eta": 0.009947129525244236, "diagnostic/loss/clic_pf/Regression_pt": 0.1321628987789154, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007760113105177879, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04215376824140549, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18485504388809204, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05770739167928696, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4212591350078583, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0013438689056783915, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18303117156028748, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07018840312957764, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42685171961784363, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9676386713981628, "metrics/particle/class_1/pid_accuracy": 0.9866883158683777, "metrics/particle/class_2/efficiency": 0.44294092059135437, "metrics/particle/class_2/pid_accuracy": 0.7782586812973022, "metrics/particle/class_3/efficiency": 0.7437439560890198, "metrics/particle/class_3/pid_accuracy": 0.9845533967018127, "metrics/particle/class_4/efficiency": 0.9186046719551086, "metrics/particle/class_4/pid_accuracy": 0.4609704613685608, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6822810769081116, "metrics/particle/count/bias_mean": -0.886274516582489, "metrics/particle/count/mae": 3.1960785388946533, "metrics/particle/count/prediction_mean": 69.11307525634766, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.048278484493494034, "metrics/particle/event/energy_response_mean": 1.0073245763778687, "metrics/particle/event/met_abs_error": 4.422399044036865, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03779202327132225, "metrics/particle/event/scalar_pt_response_mean": 1.009222149848938, "metrics/particle/event/vector_pt_closure": 0.0400027371942997, "metrics/particle/matched/delta_eta_abs_mean": 0.0077992649748921394, "metrics/particle/matched/delta_phi_abs_mean": 0.009978614747524261, "metrics/particle/matched/delta_r_mean": 0.014184625819325447, "metrics/particle/matched/energy_relative_abs_mean": 0.08033423870801926, "metrics/particle/matched/pid_accuracy": 0.9692949056625366, "metrics/particle/matched/pt_relative_abs_mean": 0.07434055209159851, "metrics/particle/matching/duplicate_fraction": 0.027869457378983498, "metrics/particle/matching/efficiency": 0.8268237709999084, "metrics/particle/matching/f1": 0.8320913910865784, "metrics/particle/matching/purity": 0.8374266028404236, "metrics/particle/matching/target_coverage_dr0p05": 0.8070570230484009, "metrics/particle/matching/target_coverage_dr0p10": 0.8605589270591736, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7361879944801331, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8634534478187561, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016664650291204453, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07876672595739365, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07543685287237167, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014215669594705105, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1317719668149948, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003374839900061488, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3212427496910095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.021488340571522713, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1826070547103882, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8593.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.281154602766037, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7640882730484009, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9642090201377869, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8477709293365479, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12165286391973495, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.374513626098633, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013195916078984737, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7171981930732727, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07019474357366562, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6404013633728027, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10382.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7994232177734375, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6642082929611206, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.481998920440674, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8974995613098145, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.028997056186199188, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5398268103599548, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.5973663330078125, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.32826972007751465, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.483016967773438, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02568839304149151, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09194513410329819, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09034810215234756, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.011726525612175465, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14688993990421295, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01971522532403469, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.905030369758606, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.028668615967035294, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1232151985168457, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1826.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7313002943992615, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5037764310836792, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 68651.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 68650.84375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2571227550506592, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3660299.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07914175093173981, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.790001392364502, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07199738919734955, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.980369567871094, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2843.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5944010615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7503094673156738, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.898892402648926, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.120947003364563, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.013829391449689865, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07560716569423676, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.735997676849365, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.16704443097114563, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.29164695739746, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012083751149475574, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08008351176977158, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07255525141954422, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016906028613448143, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12897351384162903, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005742625799030066, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10257767885923386, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0015418834518641233, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5457318425178528, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19155.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13584862649440765, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6307269930839539, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48533394932746887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.39285406470298767, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08556820452213287, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.848347187042236, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028352311346679926, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23225291073322296, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002966429805383086, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4438002109527588, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21515.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8725061416625977, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.271448016166687, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.0021209716796875, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.000151634216309, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.30236130952835083, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5506078004837036, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.831212282180786, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4827876687049866, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.748021125793457, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014254770241677761, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08043327927589417, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07449112087488174, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01580451987683773, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13096316158771515, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025698249228298664, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2156590223312378, "metrics/particle/momentum/all/accepted/pz_bias": 0.003474864410236478, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.096147060394287, "metrics/particle/momentum/all/accepted/samples": 29574.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22800278663635254, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7420291900634766, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5618.74609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5618.64111328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11039148271083832, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1047104.875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012176165357232094, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5864953398704529, "metrics/particle/momentum/all/assigned/pz_bias": 0.02870677039027214, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9302475452423096, "metrics/particle/momentum/all/assigned/samples": 34740.0, "metrics/particle/momentum/all/efficiency": 0.8266666531562805, "metrics/particle/momentum/all/event/energy_bias": -0.14306968450546265, "metrics/particle/momentum/all/event/p3_error_mean": 13.76867389678955, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.679118633270264, "metrics/particle/momentum/all/event/px_bias": -0.28719362616539, "metrics/particle/momentum/all/event/py_bias": 0.06482614576816559, "metrics/particle/momentum/all/event/pz_abs_mean": 10.410621643066406, "metrics/particle/momentum/all/event/pz_bias": 0.9781017303466797, "metrics/particle/momentum/all/event/pz_residual_rms": 18.450958251953125, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007356434129178524, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03178858757019043, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021167192608118057, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003532694187015295, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06068173423409462, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011657157447189093, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10493429005146027, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0009053210960701108, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7709599733352661, "metrics/particle/momentum/class_1/accepted/samples": 16299.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04366106912493706, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1031377762556076, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10807695239782333, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07871036976575851, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020804932340979576, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.002776861190796, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004504771437495947, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2292589694261551, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.022850530222058296, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9824635982513428, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.965981125831604, "metrics/particle/momentum/class_1/event/energy_bias": 3.5962440967559814, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.689911842346191, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.165420055389404, "metrics/particle/momentum/class_1/event/px_bias": -0.6219235062599182, "metrics/particle/momentum/class_1/event/py_bias": -0.6479036808013916, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.134878635406494, "metrics/particle/momentum/class_1/event/pz_bias": 0.7508084774017334, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.8390531539917, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035915810614824295, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19570042192935944, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19569115340709686, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03725185617804527, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23593057692050934, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03132161870598793, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5570378303527832, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.00340333953499794, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3204007148742676, "metrics/particle/momentum/class_2/accepted/samples": 1309.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.881385326385498, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6845939755439758, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 73815.6796875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 73815.1953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3490922749042511, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3795546.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07980332523584366, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0584967136383057, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.16237877309322357, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.104046821594238, "metrics/particle/momentum/class_2/assigned/samples": 2644.0, "metrics/particle/momentum/class_2/efficiency": 0.4443312883377075, "metrics/particle/momentum/class_2/event/energy_bias": -7.0889434814453125, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.071025848388672, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.63938570022583, "metrics/particle/momentum/class_2/event/px_bias": 0.09329859167337418, "metrics/particle/momentum/class_2/event/py_bias": 0.26234596967697144, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.720139503479004, "metrics/particle/momentum/class_2/event/pz_bias": -0.7290261387825012, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.259830474853516, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021830884739756584, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13664066791534424, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13781967759132385, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03969481959939003, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17968298494815826, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013968918938189745, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21890199184417725, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00857902318239212, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9819592237472534, "metrics/particle/momentum/class_3/accepted/samples": 11454.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3275391757488251, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4583463668823242, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.645641565322876, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5190072059631348, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17267216742038727, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.599000930786133, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00912744365632534, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5569785237312317, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018170049414038658, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5406665802001953, "metrics/particle/momentum/class_3/assigned/samples": 14681.0, "metrics/particle/momentum/class_3/efficiency": 0.7431871294975281, "metrics/particle/momentum/class_3/event/energy_bias": 4.238170623779297, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.354110717773438, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.859104633331299, "metrics/particle/momentum/class_3/event/px_bias": 0.06237979233264923, "metrics/particle/momentum/class_3/event/py_bias": 0.3520962595939636, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.0086669921875, "metrics/particle/momentum/class_3/event/pz_bias": 0.7844746112823486, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.430376052856445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011035568080842495, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1070212870836258, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06050015240907669, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.009574053809046745, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11002940684556961, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2644113302230835, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0066299159079790115, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.872922420501709, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11533113569021225, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.239975690841675, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6491262912750244, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5838195085525513, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.03284454345703, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.39179617166519165, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.000904789543710649, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2122468948364258, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.46942371129989624, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.81911039352417, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1631146669387817, "metrics/particle/momentum/class_4/event/px_bias": 0.16413165628910065, "metrics/particle/momentum/class_4/event/py_bias": 0.009222062304615974, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1787750720977783, "metrics/particle/momentum/class_4/event/pz_bias": 0.3391551375389099, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.284789085388184, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005154137499630451, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02029423415660858, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017094166949391365, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004067210014909506, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05333251878619194, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1898670196533203, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09507688134908676, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8063885569572449, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019096875563263893, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03920448571443558, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.035951919853687286, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012406079098582268, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1405731737613678, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5578141808509827, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.46510738134384155, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.226048469543457, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.41911694407463074, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9974104166030884, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6969653964042664, "metrics/particle/momentum/class_5/event/px_bias": 0.01491982489824295, "metrics/particle/momentum/class_5/event/py_bias": 0.08906558156013489, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6000429391860962, "metrics/particle/momentum/class_5/event/pz_bias": -0.16731026768684387, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3512237071990967, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3953843116760254, "step_valid_time": 5971.5426161289215, "step_total_time": 5972.938000917435, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002940840001886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06000436176942592, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2891, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9554455445544554, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9578163771712159, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04455445544554455, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9588723051409619, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9541254125412542, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9564929693961952, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9601990049751243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040058530453666, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07646073495776318, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9531668153434434, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9508342602892103, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04683318465655665, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9471815357301376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.951828724353256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494994438264739, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9485130936529073, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042489327546549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08225339682235522, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1181, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9190661478599221, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.921935987509758, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9204988308651598, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.078064012490242, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9151750972762646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9180327868852459, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.916601714731099, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9190661478599221} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_37000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..63ef798936a7b008940f201a6ccc50862c773233 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7528898995641462, "train": {"Classification_binary": 2.4516961574554443, "Classification": 0.020308388397097588, "Regression_pt": 0.14905841648578644, "Regression_eta": 0.01092506293207407, "Regression_sin_phi": 0.007905347272753716, "Regression_cos_phi": 0.007442473899573088, "Regression_energy": 0.1495034396648407, "Total": 8.822325706481934, "Total_unweighted": 2.79683928610757}, "train_group": {}, "valid": {"Classification_binary": 2.4836217562357583, "Classification": 0.01974932725230853, "Regression_pt": 0.12936623891194662, "Regression_eta": 0.010508904854456583, "Regression_sin_phi": 0.008009451751907667, "Regression_cos_phi": 0.008218842248121897, "Regression_energy": 0.13193581501642862, "Total": 8.34379768371582, "Total_unweighted": 2.7914103362709284, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02045198157429695, "diagnostic/loss/clic_pf/Classification_binary": 2.4916672706604004, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007855251431465149, "diagnostic/loss/clic_pf/Regression_energy": 0.13442687690258026, "diagnostic/loss/clic_pf/Regression_eta": 0.00981965847313404, "diagnostic/loss/clic_pf/Regression_pt": 0.13215605914592743, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007647428195923567, "diagnostic/regression/clic_pf/energy_pred_mean": 0.043411221355199814, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.185751810669899, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05896485224366188, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41920629143714905, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0003266003623139113, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1832425445318222, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06917113810777664, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42612922191619873, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9669827222824097, "metrics/particle/class_1/pid_accuracy": 0.9853636622428894, "metrics/particle/class_2/efficiency": 0.45466846227645874, "metrics/particle/class_2/pid_accuracy": 0.7782738208770752, "metrics/particle/class_3/efficiency": 0.7519128918647766, "metrics/particle/class_3/pid_accuracy": 0.9848645329475403, "metrics/particle/class_4/efficiency": 0.9244186282157898, "metrics/particle/class_4/pid_accuracy": 0.46960169076919556, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.756147563457489, "metrics/particle/count/bias_mean": -0.3065359592437744, "metrics/particle/count/mae": 3.0660130977630615, "metrics/particle/count/prediction_mean": 69.69281005859375, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.050284720957279205, "metrics/particle/event/energy_response_mean": 1.0145515203475952, "metrics/particle/event/met_abs_error": 4.431092262268066, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03910427913069725, "metrics/particle/event/scalar_pt_response_mean": 1.0135369300842285, "metrics/particle/event/vector_pt_closure": 0.03977886214852333, "metrics/particle/matched/delta_eta_abs_mean": 0.008587519638240337, "metrics/particle/matched/delta_phi_abs_mean": 0.009853026829659939, "metrics/particle/matched/delta_r_mean": 0.014653868973255157, "metrics/particle/matched/energy_relative_abs_mean": 0.08207044005393982, "metrics/particle/matched/pid_accuracy": 0.9690017700195312, "metrics/particle/matched/pt_relative_abs_mean": 0.0749150961637497, "metrics/particle/matching/duplicate_fraction": 0.0288567952811718, "metrics/particle/matching/efficiency": 0.8310534954071045, "metrics/particle/matching/f1": 0.8328771591186523, "metrics/particle/matching/purity": 0.8347088098526001, "metrics/particle/matching/target_coverage_dr0p05": 0.8070570230484009, "metrics/particle/matching/target_coverage_dr0p10": 0.8632853627204895, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7388584613800049, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8681873679161072, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017226260155439377, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07904386520385742, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07448502629995346, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016596337780356407, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13016383349895477, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0029069767333567142, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32067614793777466, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.011507784947752953, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2063624858856201, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8600.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28222328424453735, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8933643102645874, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0282905101776123, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9148783683776855, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12371727079153061, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.434965133666992, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012755346484482288, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7293947339057922, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.029611872509121895, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.8226261138916016, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10427.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8000743985176086, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3361026346683502, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.21730899810791, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.7314324378967285, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.10579416155815125, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.5052267909049988, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.38781213760376, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.002995520830154419, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.333293914794922, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02631637267768383, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09124970436096191, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09007353335618973, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.017309673130512238, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14640569686889648, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021288210526108742, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9840201735496521, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0017995446687564254, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.222851276397705, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1832.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7339348196983337, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5065876245498657, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 63961.1875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 63961.04296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2642432749271393, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3421011.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07584760338068008, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8389785289764404, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.19951610267162323, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.752854347229004, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2861.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5963541865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.16391253471374512, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.08923864364624, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0999395847320557, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.13440382480621338, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.06559404730796814, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.952720642089844, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.1879823207855225, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.333728790283203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01262124814093113, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08250466734170914, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0737234577536583, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013754243962466717, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13015349209308624, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007790589006617665, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11048225313425064, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015639377757906914, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.608113706111908, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19254.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1374169886112213, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6180204153060913, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4884198307991028, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.39236870408058167, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08341424912214279, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.9004292488098145, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002635594457387924, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22742296755313873, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.02188335731625557, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1423697471618652, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21627.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8770155906677246, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5631394386291504, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.615813255310059, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.622872352600098, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.09686696529388428, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3921965956687927, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.74708890914917, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.10363148152828217, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.1180419921875, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014800476841628551, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08204175531864166, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.074953094124794, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014797009527683258, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13121771812438965, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026611872017383575, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22528348863124847, "metrics/particle/momentum/all/accepted/pz_bias": -0.013588367030024529, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1412475109100342, "metrics/particle/momentum/all/accepted/samples": 29686.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22954165935516357, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7730598449707031, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5241.7080078125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5241.60302734375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11026779562234879, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 979281.25, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01165688131004572, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5913273096084595, "metrics/particle/momentum/all/assigned/pz_bias": -0.0210604015737772, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0765955448150635, "metrics/particle/momentum/all/assigned/samples": 34915.0, "metrics/particle/momentum/all/efficiency": 0.8297973275184631, "metrics/particle/momentum/all/event/energy_bias": 1.063124418258667, "metrics/particle/momentum/all/event/p3_error_mean": 13.298839569091797, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.15255880355835, "metrics/particle/momentum/all/event/px_bias": 0.14333100616931915, "metrics/particle/momentum/all/event/py_bias": 0.178624227643013, "metrics/particle/momentum/all/event/pz_abs_mean": 10.220805168151855, "metrics/particle/momentum/all/event/pz_bias": -1.2886183261871338, "metrics/particle/momentum/all/event/pz_residual_rms": 18.33082389831543, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007409736979752779, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.033776018768548965, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021569473668932915, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003820042358711362, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06162597984075546, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.00159744406118989, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11636686325073242, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0064140004105865955, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8859427571296692, "metrics/particle/momentum/class_1/accepted/samples": 16276.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04720727354288101, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12883791327476501, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.13428810238838196, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10390149801969528, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021811284124851227, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.053184509277344, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004623044282197952, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23743146657943726, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005763938184827566, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.098499298095703, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9646180272102356, "metrics/particle/momentum/class_1/event/energy_bias": 2.2411439418792725, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.251546859741211, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.885312080383301, "metrics/particle/momentum/class_1/event/px_bias": -0.32074546813964844, "metrics/particle/momentum/class_1/event/py_bias": -0.42647430300712585, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.873895645141602, "metrics/particle/momentum/class_1/event/pz_bias": 0.3918209671974182, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.186312675476074, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03749404475092888, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19427937269210815, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1952177733182907, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.027467168867588043, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23354944586753845, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029895367100834846, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5660701990127563, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.07672422379255295, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4005565643310547, "metrics/particle/momentum/class_2/accepted/samples": 1338.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8555322885513306, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6506738662719727, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 68329.5, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 68329.0546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34951457381248474, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3535967.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07244212180376053, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.949206590652466, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.07020650058984756, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.753538608551025, "metrics/particle/momentum/class_2/assigned/samples": 2678.0, "metrics/particle/momentum/class_2/efficiency": 0.4541751444339752, "metrics/particle/momentum/class_2/event/energy_bias": -5.199718475341797, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.023334503173828, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.699336528778076, "metrics/particle/momentum/class_2/event/px_bias": 0.2995934784412384, "metrics/particle/momentum/class_2/event/py_bias": 0.08038175106048584, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.480228424072266, "metrics/particle/momentum/class_2/event/pz_bias": -1.3798198699951172, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.80002212524414, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022829564288258553, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13701853156089783, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1375698298215866, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0356852263212204, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17935918271541595, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011244701454415917, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22300821542739868, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.015148215927183628, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9328398108482361, "metrics/particle/momentum/class_3/accepted/samples": 11561.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32933861017227173, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5393898487091064, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7123132944107056, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5840528011322021, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.170231431722641, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.365076065063477, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008906281553208828, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5739545226097107, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03614690527319908, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.965277910232544, "metrics/particle/momentum/class_3/assigned/samples": 14821.0, "metrics/particle/momentum/class_3/efficiency": 0.750129759311676, "metrics/particle/momentum/class_3/event/energy_bias": 4.474698543548584, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.258729934692383, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.90814208984375, "metrics/particle/momentum/class_3/event/px_bias": 0.020822256803512573, "metrics/particle/momentum/class_3/event/py_bias": 0.38769352436065674, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.7626142501831055, "metrics/particle/momentum/class_3/event/pz_bias": -0.30419012904167175, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.5665864944458, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011285840533673763, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11353921890258789, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05776343494653702, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0019619660452008247, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10916502773761749, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26784780621528625, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.02550882287323475, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8947210907936096, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11657683551311493, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.6951126456260681, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4693187177181244, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.4072626829147339, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.5118751525878906, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.35211995244026184, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05147094279527664, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0602898597717285, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.4240025281906128, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7444512844085693, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1034992933273315, "metrics/particle/momentum/class_4/event/px_bias": 0.13159915804862976, "metrics/particle/momentum/class_4/event/py_bias": 0.04176970198750496, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1526654958724976, "metrics/particle/momentum/class_4/event/pz_bias": 0.25766992568969727, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.1405463218688965, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0050676739774644375, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02154785394668579, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017251450568437576, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0035474307369440794, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05030195042490959, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17354251444339752, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.07166406512260437, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7109752893447876, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01935502514243126, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04504105821251869, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0406930148601532, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01656428538262844, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.15644539892673492, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5946524739265442, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.49557289481163025, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.268570423126221, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.02899707853794098, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0634961128234863, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7505236268043518, "metrics/particle/momentum/class_5/event/px_bias": 0.012061558663845062, "metrics/particle/momentum/class_5/event/py_bias": 0.09525356441736221, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6250119209289551, "metrics/particle/momentum/class_5/event/pz_bias": -0.2540991008281708, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.453983783721924, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3917715549468994, "step_valid_time": 6112.6696009635925, "step_total_time": 6114.061372756958, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0037322839300455, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060577633257484664, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2885, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2881, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.956882255389718, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9537190082644628, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9552980132450332, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04628099173553719, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9555555555555556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952396694214876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9539735099337748, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.956882255389718, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052710667533065, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07366694702760612, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9522960320998662, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9501779359430605, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04770396790013375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467376830892144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9509585376727597, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488434163701067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948069241011984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0072689429879296, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07442172893823451, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221789883268483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.921461897356143, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9218203033838973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07853810264385692, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9152410575427683, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9155970439517698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221789883268483} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_38000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..8b2d86448eaf08dd372ab16d7fc203b6b4b41fc0 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.8002653022550692, "train": {"Classification_binary": 2.351574420928955, "Classification": 0.019999878481030464, "Regression_pt": 0.13447508215904236, "Regression_eta": 0.007704791147261858, "Regression_sin_phi": 0.00832786038517952, "Regression_cos_phi": 0.008169317618012428, "Regression_energy": 0.13384217023849487, "Total": 8.225625038146973, "Total_unweighted": 2.6640935209579766}, "train_group": {}, "valid": {"Classification_binary": 2.471708297729492, "Classification": 0.01959100489815076, "Regression_pt": 0.1280774176120758, "Regression_eta": 0.010514310250679651, "Regression_sin_phi": 0.007980051760872206, "Regression_cos_phi": 0.008181636532147726, "Regression_energy": 0.13082082072893778, "Total": 8.288525899251303, "Total_unweighted": 2.7768735395123563, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02023426443338394, "diagnostic/loss/clic_pf/Classification_binary": 2.483366012573242, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007785606198012829, "diagnostic/loss/clic_pf/Regression_energy": 0.13270369172096252, "diagnostic/loss/clic_pf/Regression_eta": 0.00986422784626484, "diagnostic/loss/clic_pf/Regression_pt": 0.13006046414375305, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007694830186665058, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03971978649497032, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18410491943359375, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05527340993285179, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4176691770553589, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.004898405633866787, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18119828402996063, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06394612789154053, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4231118857860565, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9671417474746704, "metrics/particle/class_1/pid_accuracy": 0.986660897731781, "metrics/particle/class_2/efficiency": 0.4597429037094116, "metrics/particle/class_2/pid_accuracy": 0.7895511388778687, "metrics/particle/class_3/efficiency": 0.7593706250190735, "metrics/particle/class_3/pid_accuracy": 0.9839345812797546, "metrics/particle/class_4/efficiency": 0.9205426573753357, "metrics/particle/class_4/pid_accuracy": 0.4589473605155945, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7189409136772156, "metrics/particle/count/bias_mean": 0.14575164020061493, "metrics/particle/count/mae": 3.0594770908355713, "metrics/particle/count/prediction_mean": 70.14509582519531, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04873879626393318, "metrics/particle/event/energy_response_mean": 1.0129400491714478, "metrics/particle/event/met_abs_error": 4.282034873962402, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03785521164536476, "metrics/particle/event/scalar_pt_response_mean": 1.0098296403884888, "metrics/particle/event/vector_pt_closure": 0.03829549252986908, "metrics/particle/matched/delta_eta_abs_mean": 0.00874058622866869, "metrics/particle/matched/delta_phi_abs_mean": 0.009787216782569885, "metrics/particle/matched/delta_r_mean": 0.01472947932779789, "metrics/particle/matched/energy_relative_abs_mean": 0.08157917112112045, "metrics/particle/matched/pid_accuracy": 0.9695200324058533, "metrics/particle/matched/pt_relative_abs_mean": 0.07451587170362473, "metrics/particle/matching/duplicate_fraction": 0.02815825305879116, "metrics/particle/matching/efficiency": 0.8347697257995605, "metrics/particle/matching/f1": 0.8339015245437622, "metrics/particle/matching/purity": 0.8330351710319519, "metrics/particle/matching/target_coverage_dr0p05": 0.813350260257721, "metrics/particle/matching/target_coverage_dr0p10": 0.8677111864089966, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7432375550270081, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8718662261962891, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017316581681370735, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0799708440899849, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07493185997009277, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010669057257473469, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13079556822776794, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003000230761244893, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3261057436466217, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.033139392733573914, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1764552593231201, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8666.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2745378911495209, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9971608519554138, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0825629234313965, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9646245241165161, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1209431067109108, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.505521774291992, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012886597774922848, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7141137719154358, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08759423345327377, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.527834892272949, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10476.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8062145113945007, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6084814071655273, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.2091217041015625, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.728649377822876, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06912669539451599, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.20075033605098724, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.353470802307129, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.0110796689987183, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.119915962219238, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026018036529421806, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09271004050970078, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08989953994750977, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012657114304602146, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14613689482212067, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020945219323039055, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9294031262397766, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11047936230897903, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2083611488342285, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1862.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7636296153068542, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.0367138385772705, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10430.41015625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10430.248046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24930651485919952, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 559870.5625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0776699036359787, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8776872158050537, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1588699370622635, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.170913696289062, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2884.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6061198115348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4259982109069824, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.658417224884033, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0394871234893799, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03331102058291435, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10739253461360931, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.519519805908203, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.1643595695495605, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.788232803344727, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012642748653888702, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08141282945871353, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07298469543457031, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011988555081188679, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1288372129201889, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005698891473002732, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10624042898416519, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.001999522326514125, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5737401843070984, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19302.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13581743836402893, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6578701734542847, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5154789686203003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4179568588733673, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08424840867519379, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.079131603240967, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002168496837839484, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2278566062450409, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002957004588097334, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4093536138534546, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21674.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8792019486427307, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.339313268661499, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.666399002075195, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.713531494140625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.27055802941322327, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.1185610294342041, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7153127193450928, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.24865147471427917, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.87229061126709, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014835448004305363, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08169908821582794, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07460620254278183, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.011646955274045467, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13055141270160675, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025477707386016846, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22149626910686493, "metrics/particle/momentum/all/accepted/pz_bias": 0.017817407846450806, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1214333772659302, "metrics/particle/momentum/all/accepted/samples": 29830.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2289796769618988, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9551526308059692, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 859.2742919921875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 859.1653442382812, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10880858451128006, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 160635.078125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011588742025196552, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5913931727409363, "metrics/particle/momentum/all/assigned/pz_bias": 0.041100308299064636, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1722469329833984, "metrics/particle/momentum/all/assigned/samples": 35034.0, "metrics/particle/momentum/all/efficiency": 0.83382248878479, "metrics/particle/momentum/all/event/energy_bias": 1.5217965841293335, "metrics/particle/momentum/all/event/p3_error_mean": 13.0250244140625, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.185510635375977, "metrics/particle/momentum/all/event/px_bias": -0.16812032461166382, "metrics/particle/momentum/all/event/py_bias": 0.18958187103271484, "metrics/particle/momentum/all/event/pz_abs_mean": 9.953838348388672, "metrics/particle/momentum/all/event/pz_bias": 2.424090623855591, "metrics/particle/momentum/all/event/pz_residual_rms": 17.810298919677734, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007216871250420809, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.033154673874378204, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021732401102781296, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004137615207582712, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.062365222722291946, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001473296550102532, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11529386788606644, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0005065204459242523, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8642744421958923, "metrics/particle/momentum/class_1/accepted/samples": 16290.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.045694008469581604, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1759907305240631, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.14695748686790466, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.11675690114498138, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020920997485518456, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.898998737335205, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004385705105960369, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2404799461364746, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.018702669069170952, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4886069297790527, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9654477834701538, "metrics/particle/momentum/class_1/event/energy_bias": 2.9437246322631836, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.210121154785156, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.868015289306641, "metrics/particle/momentum/class_1/event/px_bias": -0.6269969940185547, "metrics/particle/momentum/class_1/event/py_bias": -0.28524771332740784, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.793501853942871, "metrics/particle/momentum/class_1/event/pz_bias": 0.6267679929733276, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.154687881469727, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0362686924636364, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19304496049880981, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19397079944610596, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.040774982422590256, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23299966752529144, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.027920646592974663, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4827734231948853, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.027174463495612144, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1331822872161865, "metrics/particle/momentum/class_2/accepted/samples": 1361.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8570166230201721, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6742280125617981, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11058.830078125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11058.35546875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3530709743499756, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 576607.9375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07576315104961395, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.058176040649414, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.13478553295135498, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.621138572692871, "metrics/particle/momentum/class_2/assigned/samples": 2719.0, "metrics/particle/momentum/class_2/efficiency": 0.4619823396205902, "metrics/particle/momentum/class_2/event/energy_bias": -5.382221698760986, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.41720962524414, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.391087055206299, "metrics/particle/momentum/class_2/event/px_bias": 0.05034700781106949, "metrics/particle/momentum/class_2/event/py_bias": 0.007698424160480499, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.082703590393066, "metrics/particle/momentum/class_2/event/pz_bias": 0.02586275339126587, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.217208862304688, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023243995383381844, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13672366738319397, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1358916014432907, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.028465867042541504, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17740999162197113, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011998629197478294, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2225654423236847, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0504310205578804, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0006866455078125, "metrics/particle/momentum/class_3/accepted/samples": 11668.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3273693323135376, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8961241245269775, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.152874231338501, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0191597938537598, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16592831909656525, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 56.70726776123047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008323265239596367, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.544617772102356, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05728711932897568, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4107728004455566, "metrics/particle/momentum/class_3/assigned/samples": 14898.0, "metrics/particle/momentum/class_3/efficiency": 0.757072389125824, "metrics/particle/momentum/class_3/event/energy_bias": 4.561302661895752, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.411810874938965, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.872225284576416, "metrics/particle/momentum/class_3/event/px_bias": 0.270313560962677, "metrics/particle/momentum/class_3/event/py_bias": 0.41811835765838623, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.976419448852539, "metrics/particle/momentum/class_3/event/pz_bias": 1.7742283344268799, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.944499969482422, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010567447170615196, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1060541644692421, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05707059055566788, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006991923321038485, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10326578468084335, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2527468204498291, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0038123189006000757, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8313722014427185, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11432787775993347, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.1177539825439453, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8271337747573853, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7643687129020691, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0635359138250351, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.416245460510254, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38035911321640015, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.056617315858602524, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1675082445144653, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.4195477068424225, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.676560878753662, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0571956634521484, "metrics/particle/momentum/class_4/event/px_bias": 0.11773223429918289, "metrics/particle/momentum/class_4/event/py_bias": -0.055352069437503815, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.110298752784729, "metrics/particle/momentum/class_4/event/pz_bias": 0.20657934248447418, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.095963478088379, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004983098711818457, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020517274737358093, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.016293279826641083, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005919187795370817, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04581795260310173, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16844817996025085, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0585993155837059, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6340020298957825, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.012723867781460285, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03855195268988609, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03434916213154793, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013715789653360844, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13288286328315735, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5201367139816284, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41270220279693604, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.095574378967285, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.18146152794361115, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0271573066711426, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7074323892593384, "metrics/particle/momentum/class_5/event/px_bias": 0.020483888685703278, "metrics/particle/momentum/class_5/event/py_bias": 0.10436481237411499, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6186017990112305, "metrics/particle/momentum/class_5/event/pz_bias": -0.20934756100177765, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.496648073196411, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3908727169036865, "step_valid_time": 6254.255645751953, "step_total_time": 6255.646518707275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018091748768838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06100539564823637, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9563636363636364, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9579470198675497, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04363636363636364, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540495867768595, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556291390728476, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9595356550580432, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009984333198398, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07068419779733537, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2131, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9458499778073679, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526151095216808, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9492204899777282, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047384890478319175, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9449622725255216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.951721054984354, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9483296213808463, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9458499778073679, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004564366124358, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07639794260446009, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1183, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9206225680933852, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.92421875, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9224171539961014, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07578125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9151750972762646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.91875, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9169590643274854, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9206225680933852} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_39000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..b8c289bec2f8eb73bd799f879c9275965f5e5c0f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.847640704945992, "train": {"Classification_binary": 2.384007215499878, "Classification": 0.019904686138033867, "Regression_pt": 0.12350714951753616, "Regression_eta": 0.00953635573387146, "Regression_sin_phi": 0.0072595239616930485, "Regression_cos_phi": 0.006585543043911457, "Regression_energy": 0.12419220805168152, "Total": 7.870561599731445, "Total_unweighted": 2.6749926819466054}, "train_group": {}, "valid": {"Classification_binary": 2.477803389231364, "Classification": 0.019582316279411316, "Regression_pt": 0.128045121828715, "Regression_eta": 0.010476997743050257, "Regression_sin_phi": 0.007989289239048958, "Regression_cos_phi": 0.0082187137256066, "Regression_energy": 0.13103212912877402, "Total": 8.297882715861002, "Total_unweighted": 2.78314795717597, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020168326795101166, "diagnostic/loss/clic_pf/Classification_binary": 2.48835825920105, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007808496244251728, "diagnostic/loss/clic_pf/Regression_energy": 0.13327987492084503, "diagnostic/loss/clic_pf/Regression_eta": 0.009691689163446426, "diagnostic/loss/clic_pf/Regression_pt": 0.1302967220544815, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007716732565313578, "diagnostic/regression/clic_pf/energy_pred_mean": 0.039728131145238876, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18371066451072693, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05528175085783005, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41488173604011536, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005072798579931259, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18031254410743713, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06377173960208893, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41875550150871277, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.968334436416626, "metrics/particle/class_1/pid_accuracy": 0.9874573945999146, "metrics/particle/class_2/efficiency": 0.4532025158405304, "metrics/particle/class_2/pid_accuracy": 0.7636228203773499, "metrics/particle/class_3/efficiency": 0.7645866870880127, "metrics/particle/class_3/pid_accuracy": 0.9872578978538513, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.43619248270988464, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7160243391990662, "metrics/particle/count/bias_mean": 0.2620915174484253, "metrics/particle/count/mae": 3.1013071537017822, "metrics/particle/count/prediction_mean": 70.26143646240234, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05010717734694481, "metrics/particle/event/energy_response_mean": 1.016492486000061, "metrics/particle/event/met_abs_error": 4.424014568328857, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03922215476632118, "metrics/particle/event/scalar_pt_response_mean": 1.0157712697982788, "metrics/particle/event/vector_pt_closure": 0.04012921079993248, "metrics/particle/matched/delta_eta_abs_mean": 0.00803554616868496, "metrics/particle/matched/delta_phi_abs_mean": 0.01057974062860012, "metrics/particle/matched/delta_r_mean": 0.014801879413425922, "metrics/particle/matched/energy_relative_abs_mean": 0.08290214836597443, "metrics/particle/matched/pid_accuracy": 0.9699738025665283, "metrics/particle/matched/pt_relative_abs_mean": 0.0757347047328949, "metrics/particle/matching/duplicate_fraction": 0.028037209063768387, "metrics/particle/matching/efficiency": 0.8371226787567139, "metrics/particle/matching/f1": 0.8355584144592285, "metrics/particle/matching/purity": 0.8339999914169312, "metrics/particle/matching/target_coverage_dr0p05": 0.8173279166221619, "metrics/particle/matching/target_coverage_dr0p10": 0.8697466850280762, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7420237064361572, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.87408846616745, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01650741510093212, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08199761807918549, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07676345109939575, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.020205270498991013, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13296003639698029, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002753240754827857, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32100942730903625, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.013265327550470829, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1622874736785889, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8717.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25821807980537415, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8295615911483765, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9361110329627991, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8262603878974915, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11676190793514252, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.793737411499023, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011523809283971786, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6840516924858093, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.02121840976178646, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4496724605560303, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10500.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8109591603279114, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.8831470012664795, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.756704330444336, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5766358375549316, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.12296287715435028, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3271207809448242, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.930960178375244, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.0737425088882446, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.867937088012695, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025061210617423058, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09154122322797775, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09039895981550217, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02801976539194584, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1461799144744873, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01828940212726593, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9255726337432861, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.015170907601714134, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6214230060577393, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1859.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7325897812843323, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.080082654953003, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3647.446044921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3647.290771484375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23991654813289642, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 195484.390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06780250370502472, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.771158456802368, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.028355026617646217, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.73853588104248, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2876.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6051432490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9152750968933105, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.777758598327637, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0561211109161377, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.015235312283039093, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11657686531543732, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.633840084075928, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.9055755138397217, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.711811065673828, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013185865245759487, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08206765353679657, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07337737828493118, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014837273396551609, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1289917230606079, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005181078449822962, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10610071569681168, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.012346264906227589, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5507713556289673, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19301.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13873673975467682, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.667401909828186, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.49692627787590027, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.399032860994339, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08385638147592545, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.270951747894287, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00253830524161458, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23001091182231903, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01816480979323387, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2300517559051514, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21668.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8791564106941223, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.108197569847107, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.464849472045898, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.561091899871826, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.22651003301143646, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.15223681926727295, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.5809216499328613, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5056647062301636, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.156546115875244, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01489387545734644, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08263667672872543, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07542441785335541, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017223693430423737, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13128763437271118, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0022759982384741306, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21979209780693054, "metrics/particle/momentum/all/accepted/pz_bias": -0.012790166772902012, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.185805320739746, "metrics/particle/momentum/all/accepted/samples": 29877.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22327262163162231, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7498567700386047, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 299.9272766113281, "metrics/particle/momentum/all/assigned/pt_relative_bias": 299.8210754394531, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10652323067188263, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 56001.5078125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010586691088974476, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5745996832847595, "metrics/particle/momentum/all/assigned/pz_bias": -0.019916031509637833, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.242962121963501, "metrics/particle/momentum/all/assigned/samples": 35044.0, "metrics/particle/momentum/all/efficiency": 0.8351362943649292, "metrics/particle/momentum/all/event/energy_bias": 2.0760693550109863, "metrics/particle/momentum/all/event/p3_error_mean": 13.18463134765625, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.137284755706787, "metrics/particle/momentum/all/event/px_bias": 0.33423760533332825, "metrics/particle/momentum/all/event/py_bias": 0.5959344506263733, "metrics/particle/momentum/all/event/pz_abs_mean": 10.157757759094238, "metrics/particle/momentum/all/event/pz_bias": -2.48498272895813, "metrics/particle/momentum/all/event/pz_residual_rms": 18.347482681274414, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007210166659206152, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03346956521272659, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021392300724983215, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004754847846925259, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06049973517656326, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001165572670288384, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10613561421632767, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0026426315307617188, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8036758899688721, "metrics/particle/momentum/class_1/accepted/samples": 16301.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.043159618973731995, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10616584867238998, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09807775169610977, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06950251013040543, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020506134256720543, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4156486988067627, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004267172422260046, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22837139666080475, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.007111645303666592, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.235790491104126, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9660996794700623, "metrics/particle/momentum/class_1/event/energy_bias": 2.8710503578186035, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.978141784667969, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.753175735473633, "metrics/particle/momentum/class_1/event/px_bias": -0.3411298394203186, "metrics/particle/momentum/class_1/event/py_bias": -0.23591157793998718, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.635672092437744, "metrics/particle/momentum/class_1/event/pz_bias": 0.09541717171669006, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.782950401306152, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036521103233098984, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19091501832008362, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19118662178516388, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03198010101914406, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22952942550182343, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02550637722015381, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4781633615493774, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0006449099164456129, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1601803302764893, "metrics/particle/momentum/class_2/accepted/samples": 1333.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8836435675621033, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6186911463737488, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3843.72314453125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3843.24658203125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34384164214134216, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 200717.09375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07038123160600662, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9127466678619385, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.1097530946135521, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.930638313293457, "metrics/particle/momentum/class_2/assigned/samples": 2728.0, "metrics/particle/momentum/class_2/efficiency": 0.45247793197631836, "metrics/particle/momentum/class_2/event/energy_bias": -7.3710174560546875, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.985605239868164, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.474392414093018, "metrics/particle/momentum/class_2/event/px_bias": -0.271389365196228, "metrics/particle/momentum/class_2/event/py_bias": -0.1810198277235031, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.749585151672363, "metrics/particle/momentum/class_2/event/pz_bias": -1.4647819995880127, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.40767478942871, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023398898541927338, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13903965055942535, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13879512250423431, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.040658000856637955, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18023277819156647, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012787723680958152, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2341965287923813, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.027928022667765617, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2316433191299438, "metrics/particle/momentum/class_3/accepted/samples": 11730.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31214770674705505, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.477852702140808, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5197275876998901, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3911467790603638, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1626954823732376, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 18.31531524658203, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007047452963888645, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5440223813056946, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.028109591454267502, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.1674540042877197, "metrics/particle/momentum/class_3/assigned/samples": 14899.0, "metrics/particle/momentum/class_3/efficiency": 0.7610952258110046, "metrics/particle/momentum/class_3/event/energy_bias": 7.351601600646973, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.835042953491211, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.186399459838867, "metrics/particle/momentum/class_3/event/px_bias": 0.7798196077346802, "metrics/particle/momentum/class_3/event/py_bias": 0.8957896828651428, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.235776424407959, "metrics/particle/momentum/class_3/event/pz_bias": -1.2507565021514893, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.612736701965332, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010182798840105534, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10129033774137497, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05573185160756111, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.005619959440082312, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10313702374696732, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2493387758731842, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0062222047708928585, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.848842203617096, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08927714824676514, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.137235641479492, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8588496446609497, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8060681819915771, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 29.7386531829834, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.36845338344573975, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06261312961578369, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2279081344604492, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.5201764106750488, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.674283742904663, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.052925944328308, "metrics/particle/momentum/class_4/event/px_bias": 0.130486398935318, "metrics/particle/momentum/class_4/event/py_bias": 0.016369149088859558, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.107057809829712, "metrics/particle/momentum/class_4/event/pz_bias": 0.2728496491909027, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.188998699188232, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00499112531542778, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022425035014748573, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01770150102674961, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005123295355588198, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.052010755985975266, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19981111586093903, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08091401308774948, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.870480477809906, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013995190151035786, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.036525923758745193, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0317588672041893, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008523945696651936, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11393330991268158, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5396634340286255, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.423379510641098, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.158120632171631, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.2553888261318207, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.011867880821228, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7075536251068115, "metrics/particle/momentum/class_5/event/px_bias": 0.03645072132349014, "metrics/particle/momentum/class_5/event/py_bias": 0.1007070243358612, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6034694910049438, "metrics/particle/momentum/class_5/event/pz_bias": -0.13771088421344757, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.419403553009033, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4299020767211914, "step_valid_time": 6394.755480289459, "step_total_time": 6396.185382604599, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029833871675664, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059864864237951454, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557463672391017, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9578024160185339, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04425363276089828, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9575456053067993, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534346103038309, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554856859175906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9598673300165838, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086934065731152, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07496034812141628, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2141, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9502885042166, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9528259902091678, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9515555555555556, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04717400979083222, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9510458388963062, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9497777777777778, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9502885042166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0113262585664402, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07213039450180214, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9198443579766536, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9241594996090696, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.921996879875195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07584050039093042, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9202501954652071, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9180967238689548, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9198443579766536} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_4000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd985769714ced4261e4c02c286dad7b6da351d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.18950161076369149, "train": {"Classification_binary": 3.730339765548706, "Classification": 0.0347152017056942, "Regression_pt": 0.20793764293193817, "Regression_eta": 0.018455402925610542, "Regression_sin_phi": 0.012870660983026028, "Regression_cos_phi": 0.013290432281792164, "Regression_energy": 0.2176462560892105, "Total": 13.39532470703125, "Total_unweighted": 4.235255362465978}, "train_group": {}, "valid": {"Classification_binary": 3.678985277811686, "Classification": 0.03322768211364746, "Regression_pt": 0.1943366527557373, "Regression_eta": 0.014825288206338882, "Regression_sin_phi": 0.01092413067817688, "Regression_cos_phi": 0.01102010409037272, "Regression_energy": 0.19658287366231283, "Total": 12.381276448567709, "Total_unweighted": 4.139902009318273, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03412984311580658, "diagnostic/loss/clic_pf/Classification_binary": 3.6751952171325684, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.01076301746070385, "diagnostic/loss/clic_pf/Regression_energy": 0.19846990704536438, "diagnostic/loss/clic_pf/Regression_eta": 0.013496607542037964, "diagnostic/loss/clic_pf/Regression_pt": 0.19669504463672638, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.01069650612771511, "diagnostic/regression/clic_pf/energy_pred_mean": 0.07949049025774002, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2387561947107315, "diagnostic/regression/clic_pf/energy_residual_mean": 0.09504411369562149, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5285500884056091, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.05771215632557869, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.24390587210655212, "diagnostic/regression/clic_pf/pt_residual_mean": 0.12655669450759888, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5400331020355225, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9493311047554016, "metrics/particle/class_1/pid_accuracy": 0.9839398860931396, "metrics/particle/class_2/efficiency": 0.3034506142139435, "metrics/particle/class_2/pid_accuracy": 0.405425488948822, "metrics/particle/class_3/efficiency": 0.6339045166969299, "metrics/particle/class_3/pid_accuracy": 0.9773546457290649, "metrics/particle/class_4/efficiency": 0.9399224519729614, "metrics/particle/class_4/pid_accuracy": 0.15567010641098022, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.41751527786254883, "metrics/particle/count/bias_mean": 3.245098114013672, "metrics/particle/count/mae": 4.871242046356201, "metrics/particle/count/prediction_mean": 73.24444580078125, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.08664558082818985, "metrics/particle/event/energy_response_mean": 1.0353729724884033, "metrics/particle/event/met_abs_error": 9.33454418182373, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08032459765672684, "metrics/particle/event/scalar_pt_response_mean": 1.0565541982650757, "metrics/particle/event/vector_pt_closure": 0.0746380090713501, "metrics/particle/matched/delta_eta_abs_mean": 0.017817554995417595, "metrics/particle/matched/delta_phi_abs_mean": 0.013749898411333561, "metrics/particle/matched/delta_r_mean": 0.02503119222819805, "metrics/particle/matched/energy_relative_abs_mean": 0.09024812281131744, "metrics/particle/matched/pid_accuracy": 0.9491146206855774, "metrics/particle/matched/pt_relative_abs_mean": 0.0886431559920311, "metrics/particle/matching/duplicate_fraction": 0.043894559144973755, "metrics/particle/matching/efficiency": 0.7592974901199341, "metrics/particle/matching/f1": 0.7420960664749146, "metrics/particle/matching/purity": 0.7256567478179932, "metrics/particle/matching/target_coverage_dr0p05": 0.6582601070404053, "metrics/particle/matching/target_coverage_dr0p10": 0.8058992028236389, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6682602167129517, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.819242000579834, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.028940144926309586, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08527278155088425, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08699586987495422, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03421240299940109, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14144736528396606, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.005353318993002176, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.41394174098968506, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.04210088029503822, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.4528121948242188, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7472.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.46668556332588196, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4024404287338257, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6728256940841675, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.536311149597168, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.20883987843990326, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.097420692443848, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.023937422782182693, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1819615364074707, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.16293221712112427, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.839222431182861, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10611.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.6951344013214111, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.3942396342754364, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.292787551879883, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.152271270751953, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.05283533036708832, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2073049247264862, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.617298126220703, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.0389647483825684, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.567123413085938, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0352330282330513, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09389600157737732, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09819342195987701, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.033506497740745544, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15329362452030182, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.023054754361510277, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.1084643602371216, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.04366007447242737, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.4756696224212646, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1388.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 1.0880426168441772, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.3141164779663086, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 72203.1796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 72203.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3714383840560913, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3896667.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09955372661352158, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.010895252227783, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.3103797435760498, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 13.16733455657959, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2913.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4518229067325592, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1792418956756592, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 11.436721801757812, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8155016899108887, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.021858971565961838, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11539778113365173, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 11.184379577636719, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 3.598867416381836, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 27.229543685913086, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.02276495285332203, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09152866154909134, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0879363939166069, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.03737190365791321, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14354391396045685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0009312517358921468, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13487032055854797, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01441622618585825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.681786298751831, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18255.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2121732383966446, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.051161766052246, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8088447451591492, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.7044242024421692, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1264546811580658, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.767508029937744, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0037111700512468815, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34109044075012207, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.030544793233275414, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6552584171295166, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21826.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8315113186836243, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 10.24923324584961, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.590428352355957, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.491534233093262, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5419875383377075, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7793274521827698, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.928703308105469, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.4239801168441772, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 18.432415008544922, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025104863569140434, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08992592990398407, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08820226788520813, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.036303382366895676, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1434864103794098, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.003282316029071808, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.26161086559295654, "metrics/particle/momentum/all/accepted/pz_bias": 0.02354217879474163, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2300230264663696, "metrics/particle/momentum/all/accepted/samples": 27115.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.36074575781822205, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2606781721115112, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5950.86962890625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5950.74951171875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.17137199640274048, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1118584.75, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.017680339515209198, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8959031105041504, "metrics/particle/momentum/all/assigned/pz_bias": 0.09334316849708557, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.796692371368408, "metrics/particle/momentum/all/assigned/samples": 35350.0, "metrics/particle/momentum/all/efficiency": 0.7579315304756165, "metrics/particle/momentum/all/event/energy_bias": 8.675750732421875, "metrics/particle/momentum/all/event/p3_error_mean": 25.007797241210938, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.6718111038208, "metrics/particle/momentum/all/event/px_bias": -0.5729638934135437, "metrics/particle/momentum/all/event/py_bias": -0.8712345957756042, "metrics/particle/momentum/all/event/pz_abs_mean": 18.42430305480957, "metrics/particle/momentum/all/event/pz_bias": 7.061812400817871, "metrics/particle/momentum/all/event/pz_residual_rms": 34.209686279296875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013268401846289635, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04290425777435303, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.03779304772615433, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.024515675380825996, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07385870814323425, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015013136435300112, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.14639581739902496, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01021173968911171, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8101832866668701, "metrics/particle/momentum/class_1/accepted/samples": 15986.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06523334980010986, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18570587038993835, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.22725901007652283, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1999480426311493, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03336691856384277, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.767788887023926, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0061044273898005486, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3284126818180084, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.039318349212408066, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.39209246635437, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9474307894706726, "metrics/particle/momentum/class_1/event/energy_bias": 13.828665733337402, "metrics/particle/momentum/class_1/event/p3_error_mean": 15.605399131774902, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.989314556121826, "metrics/particle/momentum/class_1/event/px_bias": -0.7135611772537231, "metrics/particle/momentum/class_1/event/py_bias": -0.9584454894065857, "metrics/particle/momentum/class_1/event/pz_abs_mean": 11.64283561706543, "metrics/particle/momentum/class_1/event/pz_bias": 2.1457481384277344, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.46163558959961, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04734046757221222, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2271513044834137, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22274690866470337, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06627107411623001, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2627584636211395, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.041065484285354614, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.899914264678955, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1826118379831314, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.187536239624023, "metrics/particle/momentum/class_2/accepted/samples": 901.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.277264952659607, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.095296859741211, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 75113.03125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 75112.3984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4953571557998657, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3974519.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.11107143014669418, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.373890399932861, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.495108425617218, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 12.03030014038086, "metrics/particle/momentum/class_2/assigned/samples": 2800.0, "metrics/particle/momentum/class_2/efficiency": 0.3058384358882904, "metrics/particle/momentum/class_2/event/energy_bias": -16.218700408935547, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.16490936279297, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.450766563415527, "metrics/particle/momentum/class_2/event/px_bias": 0.7491559982299805, "metrics/particle/momentum/class_2/event/py_bias": 0.013776659965515137, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.865914344787598, "metrics/particle/momentum/class_2/event/pz_bias": 2.5585105419158936, "metrics/particle/momentum/class_2/event/pz_residual_rms": 25.535146713256836, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.04306304454803467, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15588465332984924, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.16085244715213776, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.06648395210504532, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.2042083442211151, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002780638402327895, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.297395259141922, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03073534369468689, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2080981731414795, "metrics/particle/momentum/class_3/accepted/samples": 9710.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5308747887611389, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.498403310775757, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.8061108589172363, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.6752500534057617, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.27013811469078064, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.580961227416992, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.013744795694947243, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9006362557411194, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.08622552454471588, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.514850616455078, "metrics/particle/momentum/class_3/assigned/samples": 15133.0, "metrics/particle/momentum/class_3/efficiency": 0.6300285458564758, "metrics/particle/momentum/class_3/event/energy_bias": 13.701781272888184, "metrics/particle/momentum/class_3/event/p3_error_mean": 16.478803634643555, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.630393981933594, "metrics/particle/momentum/class_3/event/px_bias": -0.9337159395217896, "metrics/particle/momentum/class_3/event/py_bias": -0.18794864416122437, "metrics/particle/momentum/class_3/event/pz_abs_mean": 12.129630088806152, "metrics/particle/momentum/class_3/event/pz_bias": 1.8661658763885498, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.488662719726562, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017618536949157715, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0796634703874588, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05240490660071373, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.011997724883258343, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.08935922384262085, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.25905659794807434, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.012914342805743217, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.957300066947937, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10233460366725922, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5122437477111816, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.240584373474121, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1900309324264526, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 9.80878734588623, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3886723518371582, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04892468452453613, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.4669675827026367, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -1.5798536539077759, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.454214572906494, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6643390655517578, "metrics/particle/momentum/class_4/event/px_bias": 0.18875664472579956, "metrics/particle/momentum/class_4/event/py_bias": 0.1205124482512474, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4982755184173584, "metrics/particle/momentum/class_4/event/pz_bias": 0.4954434633255005, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.824413299560547, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010326508432626724, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.04024665802717209, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03977575898170471, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.01419096253812313, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0824897363781929, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0055555556900799274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.36779555678367615, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.043126557022333145, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.1778080463409424, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02507421001791954, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.049231503158807755, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04955446347594261, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.015160131268203259, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1278357058763504, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.61508709192276, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.20429323613643646, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5594520568847656, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -1.056141972541809, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.169890880584717, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.556985855102539, "metrics/particle/momentum/class_5/event/px_bias": 0.13640058040618896, "metrics/particle/momentum/class_5/event/py_bias": 0.14087045192718506, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.2042171955108643, "metrics/particle/momentum/class_5/event/pz_bias": -0.004055991768836975, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.37591028213501, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.324594259262085, "step_valid_time": 874.3779654502869, "step_total_time": 875.7025601863861, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0412833525656011, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13959549158134843, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2772, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2734, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9194029850746268, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9178807947019868, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9186412593206297, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08211920529801324, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9067993366500829, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9052980132450331, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9060480530240265, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9194029850746268, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0310869049714486, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15149689401746347, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2259, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2012, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1985, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8930315135375055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8906595838866755, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8918439716312057, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10934041611332448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8810474922325788, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8787073926516158, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8798758865248227, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8930315135375055, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0290700358283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1591597223902872, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1294, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1106, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1081, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8607003891050584, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8547140649149922, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8576967816983326, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.14528593508500773, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8412451361867704, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8353941267387944, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8383094222566886, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8607003891050584} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_40000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..240b11f5b3eab718480b1bfc1a444c1bc5cc2e45 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.895016107636915, "train": {"Classification_binary": 2.5194759368896484, "Classification": 0.021624073386192322, "Regression_pt": 0.14179688692092896, "Regression_eta": 0.010526117868721485, "Regression_sin_phi": 0.007844354957342148, "Regression_cos_phi": 0.008330298587679863, "Regression_energy": 0.14134493470191956, "Total": 8.745489120483398, "Total_unweighted": 2.8509426033124328}, "train_group": {}, "valid": {"Classification_binary": 2.4610535303751626, "Classification": 0.019439977904160816, "Regression_pt": 0.12805694341659546, "Regression_eta": 0.010421371708313623, "Regression_sin_phi": 0.007983068625132242, "Regression_cos_phi": 0.008135108898083368, "Regression_energy": 0.13053712248802185, "Total": 8.26243782043457, "Total_unweighted": 2.76562712341547, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020184680819511414, "diagnostic/loss/clic_pf/Classification_binary": 2.4706127643585205, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007708778604865074, "diagnostic/loss/clic_pf/Regression_energy": 0.13323162496089935, "diagnostic/loss/clic_pf/Regression_eta": 0.009657862596213818, "diagnostic/loss/clic_pf/Regression_pt": 0.13082465529441833, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007619329262524843, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04646212235093117, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18541261553764343, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06201574578881264, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42206037044525146, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.003500736318528652, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18325026333332062, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07234527170658112, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42814740538597107, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9674001336097717, "metrics/particle/class_1/pid_accuracy": 0.9872192740440369, "metrics/particle/class_2/efficiency": 0.4554578363895416, "metrics/particle/class_2/pid_accuracy": 0.777420163154602, "metrics/particle/class_3/efficiency": 0.755749523639679, "metrics/particle/class_3/pid_accuracy": 0.9848558306694031, "metrics/particle/class_4/efficiency": 0.9215116500854492, "metrics/particle/class_4/pid_accuracy": 0.46898001432418823, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7012194991111755, "metrics/particle/count/bias_mean": -0.24575163424015045, "metrics/particle/count/mae": 3.0954248905181885, "metrics/particle/count/prediction_mean": 69.75359344482422, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.053963541984558105, "metrics/particle/event/energy_response_mean": 1.0239059925079346, "metrics/particle/event/met_abs_error": 4.323242664337158, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04038160666823387, "metrics/particle/event/scalar_pt_response_mean": 1.0173146724700928, "metrics/particle/event/vector_pt_closure": 0.040442805737257004, "metrics/particle/matched/delta_eta_abs_mean": 0.007871816866099834, "metrics/particle/matched/delta_phi_abs_mean": 0.00938321277499199, "metrics/particle/matched/delta_r_mean": 0.01375137735158205, "metrics/particle/matched/energy_relative_abs_mean": 0.08076401054859161, "metrics/particle/matched/pid_accuracy": 0.969690203666687, "metrics/particle/matched/pt_relative_abs_mean": 0.07465636730194092, "metrics/particle/matching/duplicate_fraction": 0.028288185596466064, "metrics/particle/matching/efficiency": 0.8329862952232361, "metrics/particle/matching/f1": 0.8344510793685913, "metrics/particle/matching/purity": 0.835921049118042, "metrics/particle/matching/target_coverage_dr0p05": 0.814349353313446, "metrics/particle/matching/target_coverage_dr0p10": 0.8664973378181458, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7403430342674255, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8683741092681885, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015791000798344612, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07658850401639938, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07270459830760956, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013707307167351246, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12682020664215088, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028995592147111893, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3089348375797272, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.007024364545941353, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1496652364730835, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8622.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2675798833370209, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9040029644966125, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9324650168418884, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.821738064289093, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1259450614452362, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.517219543457031, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01387692615389824, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7442549467086792, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07199973613023758, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.844388961791992, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10449.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.802121102809906, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.800737202167511, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.814354419708252, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5101280212402344, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.06416647136211395, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.24148933589458466, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.064315319061279, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.05582641065120697, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.775793075561523, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02427654154598713, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09297307580709457, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09198151528835297, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026906704530119896, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14786459505558014, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019469983875751495, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9568667411804199, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.02527613379061222, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.602025270462036, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1849.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.709263265132904, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.463634967803955, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10688.7373046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10688.59375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2525217533111572, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 572923.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0709565207362175, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6570093631744385, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07806377112865448, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.972330570220947, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2875.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6018880009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.5273847579956055, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.1681694984436035, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.072034239768982, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03567305952310562, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.14919570088386536, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.024110794067383, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.2598748207092285, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.100770950317383, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011789670214056969, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08126816153526306, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07374726980924606, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014524882659316063, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1304127722978592, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007270083879120648, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10754690319299698, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.009645042940974236, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5701478719711304, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19257.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1340276598930359, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6429125666618347, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48986175656318665, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3946632146835327, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08388888835906982, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.1737961769104, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0023611111100763083, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22745747864246368, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.011383245699107647, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3056714534759521, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21600.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8771522045135498, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4567828178405762, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.588916778564453, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.6276116371154785, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.0684446468949318, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.08370663225650787, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.702174425125122, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.26709166169166565, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.3056640625, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013726823963224888, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08063893765211105, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07457897067070007, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015057876706123352, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13054318726062775, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025228741578757763, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21878071129322052, "metrics/particle/momentum/all/accepted/pz_bias": -0.0026384200900793076, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1835942268371582, "metrics/particle/momentum/all/accepted/samples": 29728.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22133979201316833, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7885920405387878, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 880.49609375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 880.3921508789062, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11035390943288803, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 164381.8125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011453441344201565, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.582084059715271, "metrics/particle/momentum/all/assigned/pz_bias": 0.00807507149875164, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.950798988342285, "metrics/particle/momentum/all/assigned/samples": 34924.0, "metrics/particle/momentum/all/efficiency": 0.830971360206604, "metrics/particle/momentum/all/event/energy_bias": 3.784904956817627, "metrics/particle/momentum/all/event/p3_error_mean": 13.56915283203125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.157120704650879, "metrics/particle/momentum/all/event/px_bias": 0.031394876539707184, "metrics/particle/momentum/all/event/py_bias": 0.30697840452194214, "metrics/particle/momentum/all/event/pz_abs_mean": 10.61007308959961, "metrics/particle/momentum/all/event/pz_bias": 0.0630432739853859, "metrics/particle/momentum/all/event/pz_residual_rms": 18.64516258239746, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007101988419890404, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03164750337600708, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021038206294178963, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00281522236764431, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.060403674840927124, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001351102371700108, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.103911854326725, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0008602467132732272, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7697373032569885, "metrics/particle/momentum/class_1/accepted/samples": 16283.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0450151227414608, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10107153654098511, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08722804486751556, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05710941553115845, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020980264991521835, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6878361701965332, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003911574836820364, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2211010456085205, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.012930856086313725, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.139317750930786, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9650328755378723, "metrics/particle/momentum/class_1/event/energy_bias": 3.2511119842529297, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.284363746643066, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.930377006530762, "metrics/particle/momentum/class_1/event/px_bias": -0.694108247756958, "metrics/particle/momentum/class_1/event/py_bias": -0.6007571220397949, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.862851619720459, "metrics/particle/momentum/class_1/event/pz_bias": 0.3717755675315857, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.095425605773926, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03581473231315613, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19389578700065613, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19507889449596405, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02732418105006218, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23402589559555054, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.026885734871029854, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.4836602210998535, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.04703803360462189, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.1652655601501465, "metrics/particle/momentum/class_2/accepted/samples": 1339.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8314121961593628, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7265810370445251, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11374.267578125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11373.8125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35505369305610657, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 591089.8125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07626804709434509, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9803383350372314, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.027358783408999443, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.652662754058838, "metrics/particle/momentum/class_2/assigned/samples": 2701.0, "metrics/particle/momentum/class_2/efficiency": 0.45451459288597107, "metrics/particle/momentum/class_2/event/energy_bias": -4.548673152923584, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.818137168884277, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.3743109703063965, "metrics/particle/momentum/class_2/event/px_bias": 0.18374988436698914, "metrics/particle/momentum/class_2/event/py_bias": 0.10043246299028397, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.575577735900879, "metrics/particle/momentum/class_2/event/pz_bias": -0.6579465866088867, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.14910316467285, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020683692768216133, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13677074015140533, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13732118904590607, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03783406689763069, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1786080300807953, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014664020854979753, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.23388482630252838, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0016064130468294024, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.2555110454559326, "metrics/particle/momentum/class_3/accepted/samples": 11593.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3161434531211853, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.561375379562378, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7877233028411865, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.66201913356781, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1699986457824707, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.615169525146484, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008510063402354717, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5615733861923218, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.006268980447202921, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.645940065383911, "metrics/particle/momentum/class_3/assigned/samples": 14806.0, "metrics/particle/momentum/class_3/efficiency": 0.7522060871124268, "metrics/particle/momentum/class_3/event/energy_bias": 5.936025619506836, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.340818405151367, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.751550674438477, "metrics/particle/momentum/class_3/event/px_bias": 0.36112335324287415, "metrics/particle/momentum/class_3/event/py_bias": 0.6804168224334717, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.9794840812683105, "metrics/particle/momentum/class_3/event/pz_bias": 0.2740693986415863, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.082252502441406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011205584742128849, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10011345893144608, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05604298785328865, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0014117717510089278, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10584454238414764, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24857521057128906, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.009281694889068604, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8446098566055298, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10816637426614761, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.0683162212371826, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.661616325378418, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.596846580505371, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.55646514892578, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.37918153405189514, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.055993542075157166, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1722325086593628, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.4995681047439575, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7005976438522339, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0762487649917603, "metrics/particle/momentum/class_4/event/px_bias": 0.1227826401591301, "metrics/particle/momentum/class_4/event/py_bias": 0.02188222110271454, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.12235426902771, "metrics/particle/momentum/class_4/event/pz_bias": 0.28626424074172974, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.178411960601807, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005244310945272446, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01781136356294155, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014433869160711765, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0014041955582797527, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04011248052120209, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1718902587890625, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09526634216308594, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6260871887207031, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.026979602873325348, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.035483088344335556, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03215918689966202, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008566444739699364, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12938031554222107, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5288599729537964, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4539200961589813, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.175761699676514, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3539915978908539, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9918249845504761, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6886926293373108, "metrics/particle/momentum/class_5/event/px_bias": 0.057847268879413605, "metrics/particle/momentum/class_5/event/py_bias": 0.10500403493642807, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.601380467414856, "metrics/particle/momentum/class_5/event/pz_bias": -0.2111193686723709, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3745923042297363, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4200708866119385, "step_valid_time": 6536.189502954483, "step_total_time": 6537.609574317932, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029671455079066, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05928070824248999, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9566798941798942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9581056466302368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04332010582010582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9543650793650794, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557873820168902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9595356550580432, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004432019050491, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07420726864967597, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2138, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9489569462938304, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.951067615658363, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9500111086425239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04893238434163701, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9497330960854092, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9486780715396579, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9489569462938304, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009160263182671, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0790070526967992, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1185, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1180, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221789883268483, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9228971962616822, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9225379525107046, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07710280373831775, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9182879377431906, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9190031152647975, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9186453873102375, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221789883268483} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_41000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..4ab5ecaf1ffb458196342b41d013fa72585378fc --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9423915103278377, "train": {"Classification_binary": 2.5063791275024414, "Classification": 0.01850614883005619, "Regression_pt": 0.1343209445476532, "Regression_eta": 0.009506885893642902, "Regression_sin_phi": 0.008166109211742878, "Regression_cos_phi": 0.007904921658337116, "Regression_energy": 0.132064089179039, "Total": 8.351863861083984, "Total_unweighted": 2.8168482268229127}, "train_group": {}, "valid": {"Classification_binary": 2.4635214805603027, "Classification": 0.019406816611687343, "Regression_pt": 0.1277193526426951, "Regression_eta": 0.010440949350595474, "Regression_sin_phi": 0.007997187475363413, "Regression_cos_phi": 0.008169269810120264, "Regression_energy": 0.1303293208281199, "Total": 8.25984001159668, "Total_unweighted": 2.7675843772788844, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019986920058727264, "diagnostic/loss/clic_pf/Classification_binary": 2.4763388633728027, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007697248365730047, "diagnostic/loss/clic_pf/Regression_energy": 0.1324794888496399, "diagnostic/loss/clic_pf/Regression_eta": 0.00982758030295372, "diagnostic/loss/clic_pf/Regression_pt": 0.12987197935581207, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.00772902462631464, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04500894993543625, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18431241810321808, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06056256964802742, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4188731908798218, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0037314100190997124, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1824767142534256, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07257594168186188, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4248709976673126, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.968970537185669, "metrics/particle/class_1/pid_accuracy": 0.9871168732643127, "metrics/particle/class_2/efficiency": 0.44936850666999817, "metrics/particle/class_2/pid_accuracy": 0.7643663883209229, "metrics/particle/class_3/efficiency": 0.7571074366569519, "metrics/particle/class_3/pid_accuracy": 0.9867619276046753, "metrics/particle/class_4/efficiency": 0.9312015771865845, "metrics/particle/class_4/pid_accuracy": 0.43808531761169434, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7393075227737427, "metrics/particle/count/bias_mean": -0.16078431904315948, "metrics/particle/count/mae": 3.073202610015869, "metrics/particle/count/prediction_mean": 69.83856201171875, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04805026575922966, "metrics/particle/event/energy_response_mean": 1.013840675354004, "metrics/particle/event/met_abs_error": 4.2076568603515625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03909389674663544, "metrics/particle/event/scalar_pt_response_mean": 1.0161840915679932, "metrics/particle/event/vector_pt_closure": 0.03880994766950607, "metrics/particle/matched/delta_eta_abs_mean": 0.007535502314567566, "metrics/particle/matched/delta_phi_abs_mean": 0.00913998018950224, "metrics/particle/matched/delta_r_mean": 0.01330377347767353, "metrics/particle/matched/energy_relative_abs_mean": 0.08139856159687042, "metrics/particle/matched/pid_accuracy": 0.9697678685188293, "metrics/particle/matched/pt_relative_abs_mean": 0.075526662170887, "metrics/particle/matching/duplicate_fraction": 0.028019802644848824, "metrics/particle/matching/efficiency": 0.8338919878005981, "metrics/particle/matching/f1": 0.8348507881164551, "metrics/particle/matching/purity": 0.8358117938041687, "metrics/particle/matching/target_coverage_dr0p05": 0.8174306154251099, "metrics/particle/matching/target_coverage_dr0p10": 0.8677672147750854, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.739203929901123, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8704936504364014, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015517241321504116, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07882553339004517, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.075416699051857, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.02225520834326744, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.132435142993927, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003242241684347391, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.322691947221756, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00677235284820199, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2154452800750732, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8636.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27364709973335266, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.84483802318573, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9571402072906494, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8506673574447632, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12093067914247513, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.353055953979492, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013309077359735966, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7115913033485413, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04590260609984398, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6620852947235107, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10444.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8034235835075378, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8909739255905151, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.949028015136719, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6311495304107666, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.023888347670435905, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.24357901513576508, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.123674392700195, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4520377516746521, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.225725173950195, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024297550320625305, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0903087705373764, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08983084559440613, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02664421871304512, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1463470309972763, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01838831789791584, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8982587456703186, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.0446675680577755, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.347083568572998, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1849.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7367236614227295, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4359667301177979, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 58105.859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 58105.69921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2593238055706024, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3112137.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07389334589242935, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9299488067626953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05719941854476929, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.954521179199219, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2869.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6018880009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0620471239089966, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.539398193359375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0243299007415771, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03938325494527817, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.15911926329135895, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.402032375335693, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3827688694000244, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.230451583862305, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011319632641971111, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08153470605611801, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07409542798995972, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016313092783093452, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13103139400482178, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005708947428502142, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10376998037099838, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.002382578793913126, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5404015779495239, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19268.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13575270771980286, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7385656237602234, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.562873125076294, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4664350152015686, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08399204909801483, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 11.192062377929688, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021726067643612623, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22563843429088593, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.00544464448466897, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4343223571777344, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21633.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8776533007621765, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4111452102661133, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.538990497589111, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.610077857971191, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5448426604270935, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07068509608507156, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.630289316177368, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.3951266407966614, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.4941606521606445, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013344528153538704, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08129362016916275, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07545680552721024, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01867985725402832, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13244085013866425, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024535341653972864, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21668697893619537, "metrics/particle/momentum/all/accepted/pz_bias": 0.0007328077917918563, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.146353840827942, "metrics/particle/momentum/all/accepted/samples": 29753.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22630254924297333, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8275816440582275, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4771.0146484375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4770.91015625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10942596942186356, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 891713.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011389000341296196, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5928895473480225, "metrics/particle/momentum/all/assigned/pz_bias": 0.012392997741699219, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.395111560821533, "metrics/particle/momentum/all/assigned/samples": 34946.0, "metrics/particle/momentum/all/efficiency": 0.8316701650619507, "metrics/particle/momentum/all/event/energy_bias": 1.2400721311569214, "metrics/particle/momentum/all/event/p3_error_mean": 12.96243667602539, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.218811988830566, "metrics/particle/momentum/all/event/px_bias": -0.5603375434875488, "metrics/particle/momentum/all/event/py_bias": 0.473383367061615, "metrics/particle/momentum/all/event/pz_abs_mean": 9.901758193969727, "metrics/particle/momentum/all/event/pz_bias": 0.3258576989173889, "metrics/particle/momentum/all/event/pz_residual_rms": 18.631437301635742, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007080571725964546, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03157178685069084, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021097378805279732, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0018983398331329226, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06122932583093643, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001348121790215373, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10794327408075333, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0010140532394871116, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8296409845352173, "metrics/particle/momentum/class_1/accepted/samples": 16319.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04362231492996216, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10413923114538193, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09979594498872757, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06887952983379364, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02020980231463909, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.229275941848755, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004267172422260046, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23169219493865967, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01355257723480463, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4402449131011963, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9671664834022522, "metrics/particle/momentum/class_1/event/energy_bias": 3.090566635131836, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.286998748779297, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.92878532409668, "metrics/particle/momentum/class_1/event/px_bias": -0.7672687768936157, "metrics/particle/momentum/class_1/event/py_bias": -0.39000821113586426, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.872492790222168, "metrics/particle/momentum/class_1/event/pz_bias": 0.4468887448310852, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.75866985321045, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034503158181905746, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19106446206569672, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19221043586730957, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03181532770395279, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23167239129543304, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028330780565738678, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5119636058807373, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.01643289253115654, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.2688980102539062, "metrics/particle/momentum/class_2/accepted/samples": 1306.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8951358199119568, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6690922379493713, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 61877.46484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 61876.98828125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35671862959861755, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3211628.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07832219451665878, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.108675479888916, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.043348729610443115, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.694728851318359, "metrics/particle/momentum/class_2/assigned/samples": 2694.0, "metrics/particle/momentum/class_2/efficiency": 0.44331297278404236, "metrics/particle/momentum/class_2/event/energy_bias": -7.8808064460754395, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.716073036193848, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.393269062042236, "metrics/particle/momentum/class_2/event/px_bias": 0.002547241747379303, "metrics/particle/momentum/class_2/event/py_bias": -0.16588717699050903, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.523655891418457, "metrics/particle/momentum/class_2/event/pz_bias": -0.3736458122730255, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.533644676208496, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0199740007519722, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13914550840854645, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14013294875621796, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.048647236078977585, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18233849108219147, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001205441658385098, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2231229692697525, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.002852714853361249, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.078774094581604, "metrics/particle/momentum/class_3/accepted/samples": 11614.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31836503744125366, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6892573833465576, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.890498399734497, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.768570899963379, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16831816732883453, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.683103561401367, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007617121562361717, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5534334778785706, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02717980556190014, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.062899589538574, "metrics/particle/momentum/class_3/assigned/samples": 14835.0, "metrics/particle/momentum/class_3/efficiency": 0.7535686492919922, "metrics/particle/momentum/class_3/event/energy_bias": 6.837745666503906, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.607693672180176, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.993975639343262, "metrics/particle/momentum/class_3/event/px_bias": 0.06847478449344635, "metrics/particle/momentum/class_3/event/py_bias": 0.9824906587600708, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.1950273513793945, "metrics/particle/momentum/class_3/event/pz_bias": 0.1726459413766861, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.539905548095703, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010617291554808617, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10280819982290268, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0577884279191494, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0011118454858660698, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10963306576013565, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2627865970134735, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.02127188816666603, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8861754536628723, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.0929531380534172, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.8149570226669312, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.6858211159706116, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.620803713798523, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.89266300201416, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3484000861644745, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0037281657569110394, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.049993872642517, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.6303210854530334, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7092318534851074, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.056988000869751, "metrics/particle/momentum/class_4/event/px_bias": 0.12515564262866974, "metrics/particle/momentum/class_4/event/py_bias": -0.033142268657684326, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1475098133087158, "metrics/particle/momentum/class_4/event/pz_bias": 0.29041627049446106, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.27003288269043, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004973657429218292, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019096286967396736, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015901627019047737, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005567760206758976, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04947705566883087, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1757919192314148, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.07619393616914749, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8597609400749207, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02329375222325325, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03212004527449608, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.031584594398736954, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00514755817130208, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12216689437627792, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5422866940498352, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.44323235750198364, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.231976509094238, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.17711234092712402, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8975909948348999, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6359639763832092, "metrics/particle/momentum/class_5/event/px_bias": 0.010753504931926727, "metrics/particle/momentum/class_5/event/py_bias": 0.07993048429489136, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5229875445365906, "metrics/particle/momentum/class_5/event/pz_bias": -0.21044740080833435, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.271226406097412, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4176721572875977, "step_valid_time": 6677.672735214233, "step_total_time": 6679.090408086777, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052167324631212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06048427753495467, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3032, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2884, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9592039800995025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538258575197889, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565073590210021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04617414248021108, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9565505804311775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9511873350923483, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9538614188853978, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9592039800995025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006308890138448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0732159173841963, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2128, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2126, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9445184198845983, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487293802942488, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466192170818505, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051270619705751225, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9436307146027519, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9478377173428444, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9457295373665481, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9445184198845983, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0088321056189864, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07385163208273116, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1186, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1181, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9229571984435798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9272869429241595, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9251170046801872, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0727130570758405, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9190661478599221, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9233776387802971, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9212168486739469, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9229571984435798} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_42000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..b551b010ad12b5a1e95ab3b2329a292bc28e052c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9897669130187607, "train": {"Classification_binary": 2.5202951431274414, "Classification": 0.019337762147188187, "Regression_pt": 0.13958580791950226, "Regression_eta": 0.0071931639686226845, "Regression_sin_phi": 0.00748513825237751, "Regression_cos_phi": 0.0066269272938370705, "Regression_energy": 0.14042183756828308, "Total": 8.363142013549805, "Total_unweighted": 2.840945780277252}, "train_group": {}, "valid": {"Classification_binary": 2.438950697580973, "Classification": 0.019181031733751297, "Regression_pt": 0.1269467572371165, "Regression_eta": 0.010439743598302206, "Regression_sin_phi": 0.007920816540718079, "Regression_cos_phi": 0.00809979997575283, "Regression_energy": 0.12960708141326904, "Total": 8.196708043416342, "Total_unweighted": 2.741145928079883, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019757166504859924, "diagnostic/loss/clic_pf/Classification_binary": 2.448439121246338, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007642962969839573, "diagnostic/loss/clic_pf/Regression_energy": 0.13166968524456024, "diagnostic/loss/clic_pf/Regression_eta": 0.009718986228108406, "diagnostic/loss/clic_pf/Regression_pt": 0.1291959434747696, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.0075985523872077465, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04420062527060509, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18355923891067505, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05975424870848656, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4167851209640503, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.003285467391833663, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1818440854549408, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0721299946308136, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4229982793331146, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9673802852630615, "metrics/particle/class_1/pid_accuracy": 0.987116277217865, "metrics/particle/class_2/efficiency": 0.4535408318042755, "metrics/particle/class_2/pid_accuracy": 0.77225261926651, "metrics/particle/class_3/efficiency": 0.7566332817077637, "metrics/particle/class_3/pid_accuracy": 0.9864687919616699, "metrics/particle/class_4/efficiency": 0.9273256063461304, "metrics/particle/class_4/pid_accuracy": 0.4482758641242981, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7469387650489807, "metrics/particle/count/bias_mean": -0.246405228972435, "metrics/particle/count/mae": 3.029411792755127, "metrics/particle/count/prediction_mean": 69.75294494628906, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04782048612833023, "metrics/particle/event/energy_response_mean": 1.0099536180496216, "metrics/particle/event/met_abs_error": 4.087960243225098, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03750493749976158, "metrics/particle/event/scalar_pt_response_mean": 1.0125079154968262, "metrics/particle/event/vector_pt_closure": 0.03832823038101196, "metrics/particle/matched/delta_eta_abs_mean": 0.00821878295391798, "metrics/particle/matched/delta_phi_abs_mean": 0.009667962789535522, "metrics/particle/matched/delta_r_mean": 0.014247458428144455, "metrics/particle/matched/energy_relative_abs_mean": 0.08088506013154984, "metrics/particle/matched/pid_accuracy": 0.9700803756713867, "metrics/particle/matched/pt_relative_abs_mean": 0.07530589401721954, "metrics/particle/matching/duplicate_fraction": 0.027707502245903015, "metrics/particle/matching/efficiency": 0.833238422870636, "metrics/particle/matching/f1": 0.8347075581550598, "metrics/particle/matching/purity": 0.8361818790435791, "metrics/particle/matching/target_coverage_dr0p05": 0.8141626119613647, "metrics/particle/matching/target_coverage_dr0p10": 0.8669642210006714, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7393346428871155, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8694292306900024, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016667427495121956, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07853499799966812, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07496865093708038, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018390074372291565, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1307574063539505, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003693444188684225, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32631343603134155, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.044951919466257095, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2158786058425903, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8664.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2760460674762726, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9512127637863159, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0615389347076416, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9514074325561523, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12317247688770294, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.220823287963867, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01337792631238699, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7192195057868958, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0022097104229032993, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5620601177215576, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10465.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8060284852981567, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.033521413803100586, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.552122116088867, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.546389102935791, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.18254226446151733, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.16550686955451965, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.7238359451293945, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.8418542742729187, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.657700538635254, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026062052696943283, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0914820060133934, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09028363227844238, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023203426972031593, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14632311463356018, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020021645352244377, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9291132092475891, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.08432691544294357, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.456188678741455, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1848.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7457866072654724, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9293574094772339, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10665.0546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10664.9072265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25034770369529724, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 571706.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07197496294975281, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.726573944091797, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.17202134430408478, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.349505424499512, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2876.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6015625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0671244859695435, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.790164947509766, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9932217597961426, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.05504098907113075, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.16629591584205627, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.671245574951172, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.5170378684997559, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.698522567749023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012249890714883804, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08076149225234985, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07393357902765274, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01825532875955105, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13015277683734894, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006747288280166686, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10648060590028763, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005041707772761583, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.578913152217865, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19267.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13604308664798737, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6771593689918518, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4878121316432953, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3939785063266754, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08413506299257278, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.430724143981934, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024976872373372316, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2270670235157013, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.014844819903373718, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1910662651062012, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21620.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8776077032089233, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.6970562934875488, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.654845714569092, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.655787467956543, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.3634164333343506, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.40833717584609985, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.75309157371521, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.01536746509373188, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.136538505554199, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014392287470400333, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08077899366617203, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07524937391281128, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018601596355438232, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1313890516757965, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027536184061318636, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22148974239826202, "metrics/particle/momentum/all/accepted/pz_bias": -0.02157353051006794, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1782370805740356, "metrics/particle/momentum/all/accepted/samples": 29779.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22811007499694824, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8622024655342102, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 877.959716796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 877.8566284179688, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1094934344291687, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 163974.234375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01146992389112711, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.580001950263977, "metrics/particle/momentum/all/assigned/pz_bias": -0.02399253658950329, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.928663492202759, "metrics/particle/momentum/all/assigned/samples": 34961.0, "metrics/particle/momentum/all/efficiency": 0.832396924495697, "metrics/particle/momentum/all/event/energy_bias": 0.5964102745056152, "metrics/particle/momentum/all/event/p3_error_mean": 12.985998153686523, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.008613109588623, "metrics/particle/momentum/all/event/px_bias": -0.4909176826477051, "metrics/particle/momentum/all/event/py_bias": 0.7401399612426758, "metrics/particle/momentum/all/event/pz_abs_mean": 9.974845886230469, "metrics/particle/momentum/all/event/pz_bias": -2.3742594718933105, "metrics/particle/momentum/all/event/pz_residual_rms": 17.720666885375977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0073080346919596195, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03149574249982834, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021427901461720467, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.001956577878445387, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.060029443353414536, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015948963118717074, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10912007838487625, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0039000329561531544, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8028988838195801, "metrics/particle/momentum/class_1/accepted/samples": 16302.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.045081693679094315, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.103168785572052, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11260586977005005, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08083247393369675, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.019617140293121338, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.380082368850708, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004148639738559723, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2211458832025528, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0052385153248906136, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1143500804901123, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.966158926486969, "metrics/particle/momentum/class_1/event/energy_bias": 2.2267682552337646, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.86000919342041, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.775075912475586, "metrics/particle/momentum/class_1/event/px_bias": -0.4163448214530945, "metrics/particle/momentum/class_1/event/py_bias": -0.36983078718185425, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.490466117858887, "metrics/particle/momentum/class_1/event/pz_bias": 0.34918200969696045, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.701939582824707, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03571408614516258, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19465112686157227, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19575786590576172, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.028204724192619324, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23462489247322083, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02923538163304329, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.532483458518982, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05873825028538704, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.275078058242798, "metrics/particle/momentum/class_2/accepted/samples": 1334.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8817298412322998, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.68184494972229, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11331.0732421875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11330.6064453125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35254988074302673, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 589391.1875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0783444195985794, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0532822608947754, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09898485988378525, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.060239315032959, "metrics/particle/momentum/class_2/assigned/samples": 2706.0, "metrics/particle/momentum/class_2/efficiency": 0.4528173804283142, "metrics/particle/momentum/class_2/event/energy_bias": -7.484487056732178, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.938335418701172, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.582975387573242, "metrics/particle/momentum/class_2/event/px_bias": -0.0649828091263771, "metrics/particle/momentum/class_2/event/py_bias": 0.20195308327674866, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.539702415466309, "metrics/particle/momentum/class_2/event/pz_bias": -1.4622949361801147, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.984837532043457, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022117335349321365, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13714918494224548, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.138350710272789, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.048016563057899475, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18027131259441376, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014617368578910828, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.228361114859581, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.041541893035173416, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1791956424713135, "metrics/particle/momentum/class_3/accepted/samples": 11630.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32280659675598145, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7313483953475952, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.007889747619629, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.888280987739563, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1695646345615387, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 58.93526840209961, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007952554151415825, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5428752303123474, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.038935337215662, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4161484241485596, "metrics/particle/momentum/class_3/assigned/samples": 14838.0, "metrics/particle/momentum/class_3/efficiency": 0.7546067833900452, "metrics/particle/momentum/class_3/event/energy_bias": 6.541017532348633, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.428345680236816, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.03181266784668, "metrics/particle/momentum/class_3/event/px_bias": -0.1293414682149887, "metrics/particle/momentum/class_3/event/py_bias": 0.722555935382843, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.831882476806641, "metrics/particle/momentum/class_3/event/pz_bias": -1.240890383720398, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.087971687316895, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01092553697526455, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10181323438882828, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05562647432088852, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.002394150709733367, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10380560904741287, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24818044900894165, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.004975464660674334, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8619248867034912, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10356618463993073, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.3804962635040283, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.121246099472046, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.0566506385803223, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0635359138250351, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 31.23300552368164, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3825671076774597, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.016389884054660797, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.186574935913086, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.5504062175750732, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.712244987487793, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1129924058914185, "metrics/particle/momentum/class_4/event/px_bias": 0.12093535810709, "metrics/particle/momentum/class_4/event/py_bias": 0.10818693041801453, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0987904071807861, "metrics/particle/momentum/class_4/event/pz_bias": 0.20404238998889923, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.014194011688232, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0051970952190458775, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.018298422917723656, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015385820530354977, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0024228363763540983, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04362819716334343, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18850557506084442, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.10695283114910126, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.758319079875946, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.005696257576346397, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03369851037859917, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030817748978734016, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008618281222879887, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11572573333978653, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.49560537934303284, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41584500670433044, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6435387134552, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.13648217916488647, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.958909809589386, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6515167951583862, "metrics/particle/momentum/class_5/event/px_bias": -0.001183927059173584, "metrics/particle/momentum/class_5/event/py_bias": 0.07727476954460144, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5892284512519836, "metrics/particle/momentum/class_5/event/pz_bias": -0.2242986112833023, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.390758991241455, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4906010627746582, "step_valid_time": 6818.788975954056, "step_total_time": 6820.279577970505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026872691209405, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05933577673501955, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547854785478548, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571546732837055, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04521452145214522, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9575456053067993, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9528052805280528, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551695616211745, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9595356550580432, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005280545020002, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07475570062689374, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2249, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9497554468652735, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9489115948467348, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05024455313472655, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9479768786127167, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9471346068414038, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948069241011984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067725551128286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07528346791460683, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9268482490272374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9297423887587822, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9282930631332814, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0702576112412178, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9266198282591726, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9251753702260327, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9268482490272374} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_43000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..403765b96c3c3c16703d0113fe5318044b66d7e8 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0371423157096835, "train": {"Classification_binary": 2.4175913333892822, "Classification": 0.02027714252471924, "Regression_pt": 0.13270258903503418, "Regression_eta": 0.010649425908923149, "Regression_sin_phi": 0.00749995606020093, "Regression_cos_phi": 0.008754483424127102, "Regression_energy": 0.135440394282341, "Total": 8.386358261108398, "Total_unweighted": 2.732915324624628}, "train_group": {}, "valid": {"Classification_binary": 2.4462809562683105, "Classification": 0.01921348770459493, "Regression_pt": 0.127957155307134, "Regression_eta": 0.010409307355682055, "Regression_sin_phi": 0.00807038446267446, "Regression_cos_phi": 0.00832738975683848, "Regression_energy": 0.1307043433189392, "Total": 8.257631301879883, "Total_unweighted": 2.750963024174174, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019950512796640396, "diagnostic/loss/clic_pf/Classification_binary": 2.4546422958374023, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007693212479352951, "diagnostic/loss/clic_pf/Regression_energy": 0.1322632133960724, "diagnostic/loss/clic_pf/Regression_eta": 0.009723520837724209, "diagnostic/loss/clic_pf/Regression_pt": 0.12971237301826477, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007607185281813145, "diagnostic/regression/clic_pf/energy_pred_mean": 0.029128853231668472, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18211129307746887, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04468247666954994, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4161221981048584, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.01538089383393526, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17952042818069458, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0534636452794075, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4197385311126709, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9677778482437134, "metrics/particle/class_1/pid_accuracy": 0.987080454826355, "metrics/particle/class_2/efficiency": 0.450045108795166, "metrics/particle/class_2/pid_accuracy": 0.7795038819313049, "metrics/particle/class_3/efficiency": 0.7650609016418457, "metrics/particle/class_3/pid_accuracy": 0.9864206314086914, "metrics/particle/class_4/efficiency": 0.9195736646652222, "metrics/particle/class_4/pid_accuracy": 0.4467860758304596, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7204081416130066, "metrics/particle/count/bias_mean": 0.19477124512195587, "metrics/particle/count/mae": 3.0392158031463623, "metrics/particle/count/prediction_mean": 70.1941146850586, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.048122260719537735, "metrics/particle/event/energy_response_mean": 1.0003565549850464, "metrics/particle/event/met_abs_error": 4.57283353805542, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03719264641404152, "metrics/particle/event/scalar_pt_response_mean": 1.001156210899353, "metrics/particle/event/vector_pt_closure": 0.0395575612783432, "metrics/particle/matched/delta_eta_abs_mean": 0.007747476454824209, "metrics/particle/matched/delta_phi_abs_mean": 0.009307976812124252, "metrics/particle/matched/delta_r_mean": 0.013585555367171764, "metrics/particle/matched/energy_relative_abs_mean": 0.08026022464036942, "metrics/particle/matched/pid_accuracy": 0.9703942537307739, "metrics/particle/matched/pt_relative_abs_mean": 0.07408247143030167, "metrics/particle/matching/duplicate_fraction": 0.027021238580346107, "metrics/particle/matching/efficiency": 0.8367118239402771, "metrics/particle/matching/f1": 0.8355493545532227, "metrics/particle/matching/purity": 0.8343901634216309, "metrics/particle/matching/target_coverage_dr0p05": 0.81717848777771, "metrics/particle/matching/target_coverage_dr0p10": 0.8677205443382263, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.746262788772583, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8746020197868347, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01609271764755249, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07890260219573975, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07459230720996857, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": -0.0009353102068416774, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1300411820411682, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003105590119957924, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3249213397502899, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.026882631704211235, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2217154502868652, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8694.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2831011116504669, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8184294104576111, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.878631591796875, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7454943656921387, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11880244314670563, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.87884521484375, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01344393566250801, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7211194634437561, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07771903276443481, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5907647609710693, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10488.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8088194131851196, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.0494838953018188, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.170431137084961, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6484694480895996, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.08642406761646271, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21831560134887695, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.386821746826172, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.1358544826507568, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.366786003112793, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.0241266880184412, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08872610330581665, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08659190684556961, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": -0.0036040523555129766, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1411292403936386, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017084890976548195, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8829845190048218, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05843256786465645, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.07509708404541, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1873.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7354333996772766, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.290749430656433, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10050.3505859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10050.1669921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24810735881328583, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 541644.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07260839641094208, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.782613754272461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.12632514536380768, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.784062385559082, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2906.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6097005009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1783232688903809, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.589202880859375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0375350713729858, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.08262058347463608, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.16905736923217773, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.455101490020752, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.85477614402771, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.852274894714355, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011445576325058937, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0805952325463295, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07323697209358215, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.009919589385390282, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12923766672611237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007252382929436862, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10717613995075226, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0037512595299631357, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5783368945121765, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19304.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13558614253997803, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6229529976844788, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4583359360694885, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.35436657071113586, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08384118229150772, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.620670318603516, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002770083025097847, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22549550235271454, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.005607947241514921, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2361761331558228, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21660.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8792930841445923, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.0061988309025764465, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.224040985107422, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.416125297546387, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.10525552183389664, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2686322331428528, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.372927188873291, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.05065792053937912, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.439110279083252, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013593277893960476, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.080612413585186, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07446884363889694, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.005912282969802618, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13024824857711792, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002443841891363263, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21919669210910797, "metrics/particle/momentum/all/accepted/pz_bias": 0.00906389020383358, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1151256561279297, "metrics/particle/momentum/all/accepted/samples": 29871.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22944970428943634, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7367994785308838, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 833.7268676757812, "metrics/particle/momentum/all/assigned/pt_relative_bias": 833.6075439453125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10791920870542526, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 155952.71875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011753294616937637, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5857707262039185, "metrics/particle/momentum/all/assigned/pz_bias": 0.03719079867005348, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.057621955871582, "metrics/particle/momentum/all/assigned/samples": 35054.0, "metrics/particle/momentum/all/efficiency": 0.8349685668945312, "metrics/particle/momentum/all/event/energy_bias": -2.2216081619262695, "metrics/particle/momentum/all/event/p3_error_mean": 13.039538383483887, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.07399320602417, "metrics/particle/momentum/all/event/px_bias": 0.06378912925720215, "metrics/particle/momentum/all/event/py_bias": 0.11874072253704071, "metrics/particle/momentum/all/event/pz_abs_mean": 10.039320945739746, "metrics/particle/momentum/all/event/pz_bias": 1.9399726390838623, "metrics/particle/momentum/all/event/pz_residual_rms": 17.555139541625977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007142286282032728, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.031683385372161865, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021131640300154686, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0028389464132487774, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06075490266084671, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001413210411556065, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10950063169002533, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.008436940610408783, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8239992260932922, "metrics/particle/momentum/class_1/accepted/samples": 16275.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.045776549726724625, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10385526716709137, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11563292145729065, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08364291489124298, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021870555356144905, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.3573808670043945, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004445234779268503, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2316219061613083, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.004144915379583836, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.200277805328369, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9645587801933289, "metrics/particle/momentum/class_1/event/energy_bias": 1.9201128482818604, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.899484634399414, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.652861595153809, "metrics/particle/momentum/class_1/event/px_bias": -0.5820876955986023, "metrics/particle/momentum/class_1/event/py_bias": -0.5476717948913574, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.669332504272461, "metrics/particle/momentum/class_1/event/pz_bias": 0.15849417448043823, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.35583782196045, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035459354519844055, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20031708478927612, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2014118880033493, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.044193509966135025, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24136309325695038, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02963525801897049, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6035208702087402, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.05559855327010155, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3328795433044434, "metrics/particle/momentum/class_2/accepted/samples": 1316.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8875058889389038, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6862786412239075, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10747.359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10746.84765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.361796110868454, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 560166.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07802723348140717, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.075209379196167, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2193606197834015, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.443357944488525, "metrics/particle/momentum/class_2/assigned/samples": 2717.0, "metrics/particle/momentum/class_2/efficiency": 0.4467073976993561, "metrics/particle/momentum/class_2/event/energy_bias": -7.546432018280029, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.78372573852539, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.6606059074401855, "metrics/particle/momentum/class_2/event/px_bias": 0.267495721578598, "metrics/particle/momentum/class_2/event/py_bias": -0.1307622790336609, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.385702133178711, "metrics/particle/momentum/class_2/event/pz_bias": 0.5236666202545166, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.24461841583252, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02026766911149025, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13523359596729279, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1354159265756607, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.01570328325033188, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17627663910388947, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009345000144094229, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21549558639526367, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.03141672536730766, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9717429876327515, "metrics/particle/momentum/class_3/accepted/samples": 11771.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32273849844932556, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4753265380859375, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.525032877922058, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3758622407913208, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16111519932746887, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.396472930908203, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008310435339808464, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5387099981307983, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.060235776007175446, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.48122239112854, "metrics/particle/momentum/class_3/assigned/samples": 14921.0, "metrics/particle/momentum/class_3/efficiency": 0.7637555003166199, "metrics/particle/momentum/class_3/event/energy_bias": 4.273214340209961, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.651812553405762, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.988894939422607, "metrics/particle/momentum/class_3/event/px_bias": 0.12204034626483917, "metrics/particle/momentum/class_3/event/py_bias": 0.6630125045776367, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.164772987365723, "metrics/particle/momentum/class_3/event/pz_bias": 1.3205662965774536, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.158113479614258, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010589155368506908, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10083696991205215, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05659406632184982, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006442916579544544, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10701611638069153, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2696566879749298, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.051648885011672974, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9144832491874695, "metrics/particle/momentum/class_4/accepted/samples": 332.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11103223264217377, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.527191162109375, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4815150201320648, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.41867491602897644, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.628969430923462, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3656391501426697, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09019379317760468, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.132683515548706, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9171270728111267, "metrics/particle/momentum/class_4/event/energy_bias": -0.5059258937835693, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6922075748443604, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1044948101043701, "metrics/particle/momentum/class_4/event/px_bias": 0.19284696877002716, "metrics/particle/momentum/class_4/event/py_bias": 0.04409834370017052, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0812405347824097, "metrics/particle/momentum/class_4/event/pz_bias": 0.17101308703422546, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.023224353790283, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005950511433184147, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01919015310704708, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015328027307987213, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008916481398046017, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.050039876252412796, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.16465353965759277, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.10038342326879501, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6863427758216858, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.020108364522457123, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03688830882310867, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.032971546053886414, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010394357144832611, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12280089408159256, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5488312244415283, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.48632675409317017, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.182652950286865, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.3625773787498474, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.930080235004425, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.653839647769928, "metrics/particle/momentum/class_5/event/px_bias": 0.06349381804466248, "metrics/particle/momentum/class_5/event/py_bias": 0.0900639072060585, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5614281296730042, "metrics/particle/momentum/class_5/event/pz_bias": -0.23376750946044922, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.327089309692383, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4297688007354736, "step_valid_time": 6959.952454805374, "step_total_time": 6961.382223844528, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986099011591374, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05980256829247266, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.957010582010582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9584368272892863, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04298941798941799, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956882255389718, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540343915343915, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9554562013578407, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9598673300165838, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9963275725994217, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07152402705335359, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2238, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9511762094984465, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9575513851653262, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9543531507459363, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04244861483467382, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9539767649687221, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.950790469828546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9511762094984465, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982855192989707, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07751624113467748, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1189, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9268482490272374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9297423887587822, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9282930631332814, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0702576112412178, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9252918287937744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9281811085089774, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.926734216679657, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9268482490272374} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_44000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..dc71ebf904526e379f5619e4ed0764a9331a6ab2 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0845177184006065, "train": {"Classification_binary": 2.4506120681762695, "Classification": 0.019072065129876137, "Regression_pt": 0.12335219234228134, "Regression_eta": 0.010418575257062912, "Regression_sin_phi": 0.007442749571055174, "Regression_cos_phi": 0.008404053747653961, "Regression_energy": 0.12239116430282593, "Total": 8.03277587890625, "Total_unweighted": 2.741692868527025}, "train_group": {}, "valid": {"Classification_binary": 2.429279327392578, "Classification": 0.01899840186039607, "Regression_pt": 0.1259616216023763, "Regression_eta": 0.010322770724693934, "Regression_sin_phi": 0.007955832406878471, "Regression_cos_phi": 0.00813035729030768, "Regression_energy": 0.12873618801434836, "Total": 8.154233932495117, "Total_unweighted": 2.729384499291579, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019658155739307404, "diagnostic/loss/clic_pf/Classification_binary": 2.4386298656463623, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007614608388394117, "diagnostic/loss/clic_pf/Regression_energy": 0.13111381232738495, "diagnostic/loss/clic_pf/Regression_eta": 0.009677352383732796, "diagnostic/loss/clic_pf/Regression_pt": 0.12835806608200073, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007636885158717632, "diagnostic/regression/clic_pf/energy_pred_mean": 0.036180030554533005, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1808260977268219, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05173365771770477, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4112505316734314, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.006780888885259628, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17850464582443237, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06206365302205086, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4162316918373108, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.96706223487854, "metrics/particle/class_1/pid_accuracy": 0.987564206123352, "metrics/particle/class_2/efficiency": 0.4518493413925171, "metrics/particle/class_2/pid_accuracy": 0.7831295132637024, "metrics/particle/class_3/efficiency": 0.7635952234268188, "metrics/particle/class_3/pid_accuracy": 0.9861969947814941, "metrics/particle/class_4/efficiency": 0.9253876209259033, "metrics/particle/class_4/pid_accuracy": 0.4617801010608673, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.723014235496521, "metrics/particle/count/bias_mean": -0.054901961237192154, "metrics/particle/count/mae": 2.9790849685668945, "metrics/particle/count/prediction_mean": 69.94444274902344, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04581908509135246, "metrics/particle/event/energy_response_mean": 1.0047576427459717, "metrics/particle/event/met_abs_error": 4.106119155883789, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0354728065431118, "metrics/particle/event/scalar_pt_response_mean": 1.007399320602417, "metrics/particle/event/vector_pt_closure": 0.03780687600374222, "metrics/particle/matched/delta_eta_abs_mean": 0.0074089206755161285, "metrics/particle/matched/delta_phi_abs_mean": 0.009586711414158344, "metrics/particle/matched/delta_r_mean": 0.013574311509728432, "metrics/particle/matched/energy_relative_abs_mean": 0.08044677972793579, "metrics/particle/matched/pid_accuracy": 0.9708142280578613, "metrics/particle/matched/pt_relative_abs_mean": 0.07447054982185364, "metrics/particle/matching/duplicate_fraction": 0.026575714349746704, "metrics/particle/matching/efficiency": 0.8359555006027222, "metrics/particle/matching/f1": 0.8362834453582764, "metrics/particle/matching/purity": 0.8366116881370544, "metrics/particle/matching/target_coverage_dr0p05": 0.8184576630592346, "metrics/particle/matching/target_coverage_dr0p10": 0.8680099844932556, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7447688579559326, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8733041286468506, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015767667442560196, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07920622825622559, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07515518367290497, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013256501406431198, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13050809502601624, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028735632076859474, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31600669026374817, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.010254684835672379, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1740624904632568, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8700.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26906412839889526, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8344871997833252, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9810695648193359, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8631742000579834, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1195538192987442, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.176755905151367, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012965964153409004, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6987239718437195, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04899582266807556, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5413029193878174, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10489.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8093776106834412, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.25231367349624634, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.622746467590332, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.560805559158325, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09132353216409683, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.28388363122940063, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.806151390075684, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.030477792024612427, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.644041061401367, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02330518141388893, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08792554587125778, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08611958473920822, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012139718048274517, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14089025557041168, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02100161463022232, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8932764530181885, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.01264827698469162, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.15053653717041, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1857.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7155908346176147, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2692182064056396, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 10527.98828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 10527.828125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24279263615608215, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 564751.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07363668084144592, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6940805912017822, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.05508513003587723, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.634243965148926, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2879.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6044921875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2357959747314453, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.458535194396973, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9525385499000549, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03013106994330883, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11862863600254059, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.347294807434082, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.10388487577438354, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.531074523925781, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011795771308243275, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.07987043261528015, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07276143878698349, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015256849117577076, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12799741327762604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006738893571309745, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10654880851507187, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.008166641928255558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5766803622245789, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19291.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1331816464662552, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6602974534034729, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.49173828959465027, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3959561586380005, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08212783932685852, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.288788795471191, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002079770667478442, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21278560161590576, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.008283180184662342, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0893945693969727, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21637.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8787009119987488, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7672817707061768, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.333195686340332, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.429035186767578, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.14441299438476562, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2999929189682007, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.572730779647827, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.19638130068778992, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.916356086730957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013669546693563461, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08017797768115997, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07429023832082748, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014479861594736576, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12956927716732025, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025797372218221426, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21654736995697021, "metrics/particle/momentum/all/accepted/pz_bias": -0.001502247410826385, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1109704971313477, "metrics/particle/momentum/all/accepted/samples": 29848.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2217983454465866, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7625731229782104, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 866.4765014648438, "metrics/particle/momentum/all/assigned/pt_relative_bias": 866.3687744140625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10655620694160461, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 161961.984375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01122696790844202, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5624686479568481, "metrics/particle/momentum/all/assigned/pz_bias": 0.01409181859344244, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.966501474380493, "metrics/particle/momentum/all/assigned/samples": 35005.0, "metrics/particle/momentum/all/efficiency": 0.8343256711959839, "metrics/particle/momentum/all/event/energy_bias": -1.216200590133667, "metrics/particle/momentum/all/event/p3_error_mean": 12.533753395080566, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.939781665802002, "metrics/particle/momentum/all/event/px_bias": 0.26586759090423584, "metrics/particle/momentum/all/event/py_bias": 0.10251936316490173, "metrics/particle/momentum/all/event/pz_abs_mean": 9.632360458374023, "metrics/particle/momentum/all/event/pz_bias": 0.26978835463523865, "metrics/particle/momentum/all/event/pz_residual_rms": 16.79721450805664, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007116770837455988, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03162534162402153, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02145640179514885, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0049009728245437145, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06049629673361778, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014124293811619282, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10947458446025848, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007605124264955521, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8390563130378723, "metrics/particle/momentum/class_1/accepted/samples": 16284.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.045781683176755905, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10661226511001587, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.13039308786392212, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10194361954927444, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020330745726823807, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.899608612060547, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004504771437495947, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2308705747127533, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0031396890990436077, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3061349391937256, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9650921821594238, "metrics/particle/momentum/class_1/event/energy_bias": 3.0980286598205566, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.888900756835938, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.725253105163574, "metrics/particle/momentum/class_1/event/px_bias": -0.22767895460128784, "metrics/particle/momentum/class_1/event/py_bias": -0.36131978034973145, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.531983852386475, "metrics/particle/momentum/class_1/event/pz_bias": -0.08903506398200989, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.775117874145508, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03497064486145973, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19360782206058502, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19596543908119202, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02973048947751522, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23482179641723633, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03094339556992054, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.51036536693573, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0031183839309960604, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.207202434539795, "metrics/particle/momentum/class_2/accepted/samples": 1325.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8513898849487305, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6153866648674011, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 11137.251953125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 11136.76171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3480338156223297, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 580916.6875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07607497274875641, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.916766405105591, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.09107769280672073, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.06126594543457, "metrics/particle/momentum/class_2/assigned/samples": 2721.0, "metrics/particle/momentum/class_2/efficiency": 0.44976240396499634, "metrics/particle/momentum/class_2/event/energy_bias": -7.769684314727783, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.401180267333984, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.344941139221191, "metrics/particle/momentum/class_2/event/px_bias": 0.238939568400383, "metrics/particle/momentum/class_2/event/py_bias": -0.16777637600898743, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.143424987792969, "metrics/particle/momentum/class_2/event/pz_bias": -0.3873109817504883, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.904293060302734, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020567897707223892, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13508747518062592, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13522617518901825, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.033041294664144516, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17618562281131744, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011085529113188386, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21878108382225037, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.008587255142629147, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9855003952980042, "metrics/particle/momentum/class_3/accepted/samples": 11727.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31148454546928406, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.503779649734497, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5878101587295532, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4577698707580566, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16228394210338593, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.872207641601562, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007196180056780577, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5124059319496155, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.020826101303100586, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3014535903930664, "metrics/particle/momentum/class_3/assigned/samples": 14869.0, "metrics/particle/momentum/class_3/efficiency": 0.760900616645813, "metrics/particle/momentum/class_3/event/energy_bias": 4.153974533081055, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.847380638122559, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.63385534286499, "metrics/particle/momentum/class_3/event/px_bias": 0.12191261351108551, "metrics/particle/momentum/class_3/event/py_bias": 0.5631284713745117, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.515097618103027, "metrics/particle/momentum/class_3/event/pz_bias": 0.6887396574020386, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.337230682373047, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01078911405056715, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09975655376911163, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.057016029953956604, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007384165655821562, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10717921704053879, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23790191113948822, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0014220859156921506, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8244431614875793, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11065741628408432, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.3598899841308594, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.020866870880127, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.965676188468933, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.463817596435547, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3753695487976074, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08569379150867462, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3020992279052734, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.4808800220489502, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6337382793426514, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0274689197540283, "metrics/particle/momentum/class_4/event/px_bias": 0.10558304190635681, "metrics/particle/momentum/class_4/event/py_bias": -0.015690576285123825, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0832185745239258, "metrics/particle/momentum/class_4/event/pz_bias": 0.2867596447467804, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.097895622253418, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00553375668823719, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023716246709227562, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020005732774734497, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010313861072063446, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.057323962450027466, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1939694732427597, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.10079149156808853, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.855438768863678, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019312255084514618, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03719336539506912, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03353380784392357, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013199768029153347, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12354215234518051, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5649943947792053, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.473352313041687, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.245851516723633, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.21763981878757477, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9389091730117798, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6630266904830933, "metrics/particle/momentum/class_5/event/px_bias": 0.027111336588859558, "metrics/particle/momentum/class_5/event/py_bias": 0.08417760580778122, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5547767877578735, "metrics/particle/momentum/class_5/event/pz_bias": -0.22936487197875977, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.305150270462036, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3967900276184082, "step_valid_time": 7100.751281738281, "step_total_time": 7102.148072004318, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0024156815988499, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05774046193656446, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2875, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552238805970149, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501814582645991, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9526959973536222, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04981854173540086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9535655058043118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9485318376773342, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9510420112471055, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552238805970149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020057664399538, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06950504320986584, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9558626839054837, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9537366548042705, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04413731609451627, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9498446515756769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.954079358002675, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9519572953736655, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516200621393698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0030652905424673, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07337990322564403, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1183, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9245136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9259547934528449, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9252336448598131, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0740452065471551, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9206225680933852, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9220576773187841, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9213395638629284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9245136186770428} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_45000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..eef21ba74ce93b6157e272872f622ad5d6d2327b --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131893121091529, "train": {"Classification_binary": 2.445176601409912, "Classification": 0.017629513517022133, "Regression_pt": 0.12742172181606293, "Regression_eta": 0.00726856105029583, "Regression_sin_phi": 0.008238292299211025, "Regression_cos_phi": 0.008071745745837688, "Regression_energy": 0.1253422647714615, "Total": 7.990480899810791, "Total_unweighted": 2.739148700609803}, "train_group": {}, "valid": {"Classification_binary": 2.431805451711019, "Classification": 0.019048822422822315, "Regression_pt": 0.12568366527557373, "Regression_eta": 0.010345435390869776, "Regression_sin_phi": 0.00791113389035066, "Regression_cos_phi": 0.008076716214418411, "Regression_energy": 0.12820489207903543, "Total": 8.141749699910482, "Total_unweighted": 2.7310761169840894, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019639955833554268, "diagnostic/loss/clic_pf/Classification_binary": 2.441685438156128, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007592757232487202, "diagnostic/loss/clic_pf/Regression_energy": 0.13083824515342712, "diagnostic/loss/clic_pf/Regression_eta": 0.009669847786426544, "diagnostic/loss/clic_pf/Regression_pt": 0.12843289971351624, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007575913332402706, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04805716499686241, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18400190770626068, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06361078470945358, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4169720709323883, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.005098987836390734, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18172383308410645, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07394352555274963, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42288917303085327, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9663863778114319, "metrics/particle/class_1/pid_accuracy": 0.9872058629989624, "metrics/particle/class_2/efficiency": 0.45230039954185486, "metrics/particle/class_2/pid_accuracy": 0.7743704915046692, "metrics/particle/class_3/efficiency": 0.7616769075393677, "metrics/particle/class_3/pid_accuracy": 0.9874638915061951, "metrics/particle/class_4/efficiency": 0.9176356792449951, "metrics/particle/class_4/pid_accuracy": 0.4677930176258087, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7342799305915833, "metrics/particle/count/bias_mean": 0.13006536662578583, "metrics/particle/count/mae": 3.0019607543945312, "metrics/particle/count/prediction_mean": 70.12940979003906, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.049946822226047516, "metrics/particle/event/energy_response_mean": 1.0217779874801636, "metrics/particle/event/met_abs_error": 4.300572872161865, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040753498673439026, "metrics/particle/event/scalar_pt_response_mean": 1.0218755006790161, "metrics/particle/event/vector_pt_closure": 0.0392768494784832, "metrics/particle/matched/delta_eta_abs_mean": 0.007310105487704277, "metrics/particle/matched/delta_phi_abs_mean": 0.009115013293921947, "metrics/particle/matched/delta_r_mean": 0.013113771565258503, "metrics/particle/matched/energy_relative_abs_mean": 0.08190253376960754, "metrics/particle/matched/pid_accuracy": 0.9708629250526428, "metrics/particle/matched/pt_relative_abs_mean": 0.07564713060855865, "metrics/particle/matching/duplicate_fraction": 0.028649182990193367, "metrics/particle/matching/efficiency": 0.8347883820533752, "metrics/particle/matching/f1": 0.8340135216712952, "metrics/particle/matching/purity": 0.8332401514053345, "metrics/particle/matching/target_coverage_dr0p05": 0.8176546692848206, "metrics/particle/matching/target_coverage_dr0p10": 0.8681500554084778, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7412954568862915, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8713806867599487, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01536872424185276, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07855579257011414, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07471844553947449, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022871170192956924, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12957122921943665, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0024204703513532877, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3073899745941162, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00013118759670760483, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.13809072971344, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8676.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27745822072029114, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7997542023658752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.812843382358551, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7054255604743958, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12423722445964813, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.560381889343262, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012776506133377552, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7123121023178101, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.029593033716082573, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.621580123901367, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10488.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8071448802947998, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.7112723588943481, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.819148540496826, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6206793785095215, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.13661333918571472, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21523168683052063, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.998554229736328, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.47170913219451904, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.951760292053223, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024680305272340775, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09375543147325516, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09293688833713531, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02903863787651062, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14974138140678406, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01983914151787758, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9604227542877197, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.006013191770762205, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.40269136428833, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1865.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7218379378318787, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2419466972351074, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3265.8564453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3265.711181640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24784259498119354, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 175652.984375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07283396273851395, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7573487758636475, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.043154194951057434, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.475305557250977, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2897.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6070963740348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.1365908980369568, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.446761131286621, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9913567304611206, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.011303529143333435, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.15868550539016724, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.3163065910339355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.022363141179084778, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.849011421203613, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01116136834025383, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0819842666387558, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07422727346420288, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017530176788568497, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1305886209011078, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005712505080737174, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10596347600221634, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0049224114045500755, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5504458546638489, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19256.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14069455862045288, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6454197764396667, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.49779632687568665, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.40417909622192383, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08518005907535553, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.59196662902832, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002446906641125679, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22997625172138214, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0004956863704137504, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1423208713531494, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21660.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8771066665649414, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.4019203186035156, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.603904724121094, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.680712699890137, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.14003153145313263, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.1417108029127121, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.64030122756958, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2743578851222992, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.848771095275879, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01323257852345705, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08172275871038437, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07554133236408234, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.019805632531642914, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13157649338245392, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002315669320523739, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2180936485528946, "metrics/particle/momentum/all/accepted/pz_bias": -0.0028428880032151937, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1391361951828003, "metrics/particle/momentum/all/accepted/samples": 29797.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22966445982456207, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7409197688102722, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 270.5234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 270.42138671875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11031530797481537, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 50502.94921875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01135682687163353, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5832516551017761, "metrics/particle/momentum/all/assigned/pz_bias": 0.012730088084936142, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.966684103012085, "metrics/particle/momentum/all/assigned/samples": 35045.0, "metrics/particle/momentum/all/efficiency": 0.8329000473022461, "metrics/particle/momentum/all/event/energy_bias": 3.9766016006469727, "metrics/particle/momentum/all/event/p3_error_mean": 13.204750061035156, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.205604553222656, "metrics/particle/momentum/all/event/px_bias": -0.2879483997821808, "metrics/particle/momentum/all/event/py_bias": 0.23220637440681458, "metrics/particle/momentum/all/event/pz_abs_mean": 10.148124694824219, "metrics/particle/momentum/all/event/pz_bias": 0.7684301137924194, "metrics/particle/momentum/all/event/pz_residual_rms": 17.471059799194336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006983582396060228, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.032900456339120865, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02187829278409481, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006473463959991932, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.061668481677770615, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011680088937282562, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1080266684293747, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.004289695061743259, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8019188642501831, "metrics/particle/momentum/class_1/accepted/samples": 16267.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.046496711671352386, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11857998371124268, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1094987615942955, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.0820372924208641, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02163219265639782, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.8835909366607666, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089373629540205, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2255236953496933, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008700069971382618, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.040245532989502, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9640846252441406, "metrics/particle/momentum/class_1/event/energy_bias": 3.32844614982605, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.96922492980957, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.790313720703125, "metrics/particle/momentum/class_1/event/px_bias": -0.59227055311203, "metrics/particle/momentum/class_1/event/py_bias": -0.5146439671516418, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.602029323577881, "metrics/particle/momentum/class_1/event/pz_bias": 0.2161969244480133, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.341571807861328, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035195909440517426, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19146548211574554, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19510376453399658, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.007336291950196028, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23517480492591858, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028722601011395454, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5459153652191162, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.08791571855545044, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4400885105133057, "metrics/particle/momentum/class_2/accepted/samples": 1323.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.90340656042099, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7125727534294128, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3461.381103515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3460.91943359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3590776026248932, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 180879.53125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07430453598499298, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0200798511505127, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.01897384412586689, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.073720932006836, "metrics/particle/momentum/class_2/assigned/samples": 2732.0, "metrics/particle/momentum/class_2/efficiency": 0.44908350706100464, "metrics/particle/momentum/class_2/event/energy_bias": -5.590947151184082, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.930855751037598, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.525985240936279, "metrics/particle/momentum/class_2/event/px_bias": 0.16755886375904083, "metrics/particle/momentum/class_2/event/py_bias": -0.0439969077706337, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.640830993652344, "metrics/particle/momentum/class_2/event/pz_bias": 0.08450937271118164, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.957293510437012, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019630281254649162, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13747356832027435, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13808788359165192, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04191484674811363, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1795155555009842, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001025640987791121, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2206811159849167, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.010296886786818504, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0212002992630005, "metrics/particle/momentum/class_3/accepted/samples": 11700.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3187577724456787, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.431694746017456, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.44596266746521, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3232916593551636, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16715897619724274, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.157485008239746, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00819011777639389, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5452083945274353, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02265368029475212, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.5562331676483154, "metrics/particle/momentum/class_3/assigned/samples": 14896.0, "metrics/particle/momentum/class_3/efficiency": 0.7591487169265747, "metrics/particle/momentum/class_3/event/energy_bias": 6.868139266967773, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.185497283935547, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.66367769241333, "metrics/particle/momentum/class_3/event/px_bias": -0.030570723116397858, "metrics/particle/momentum/class_3/event/py_bias": 0.6513665914535522, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.898316860198975, "metrics/particle/momentum/class_3/event/pz_bias": 0.45305025577545166, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.019143104553223, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010761250741779804, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1034035012125969, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.054168909788131714, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007763752248138189, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09923764318227768, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2449302077293396, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0058271875604987144, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8181377649307251, "metrics/particle/momentum/class_4/accepted/samples": 329.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.12225537747144699, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.890110731124878, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.680185079574585, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6225517988204956, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.07182320207357407, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.220998764038086, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.005524862091988325, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.44947245717048645, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.017817925661802292, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.608044147491455, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.908839762210846, "metrics/particle/momentum/class_4/event/energy_bias": -0.43732041120529175, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6356923580169678, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0532047748565674, "metrics/particle/momentum/class_4/event/px_bias": 0.15793845057487488, "metrics/particle/momentum/class_4/event/py_bias": 0.05952829122543335, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0527043342590332, "metrics/particle/momentum/class_4/event/pz_bias": 0.18834109604358673, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.960602760314941, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0051138438284397125, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023214146494865417, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01931813918054104, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008946914225816727, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05505912005901337, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18803146481513977, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09647751599550247, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8427525162696838, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.019083166494965553, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03977315500378609, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03590907156467438, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01496588159352541, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12861472368240356, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5483658313751221, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4588240683078766, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.2121429443359375, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.19171559810638428, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9524543881416321, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6608401536941528, "metrics/particle/momentum/class_5/event/px_bias": 0.009395554661750793, "metrics/particle/momentum/class_5/event/py_bias": 0.07995231449604034, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5698975920677185, "metrics/particle/momentum/class_5/event/pz_bias": -0.17366750538349152, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.384500026702881, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4360299110412598, "step_valid_time": 7241.5582683086395, "step_total_time": 7242.994298696518, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0072236782206632, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06134566589226087, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.953243332235759, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.956708526107072, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046756667764241026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956882255389718, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9499506091537702, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534038334434898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9601990049751243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0108590239407622, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07320820284945229, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9537366548042705, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952677182848256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046263345195729534, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9506227758007118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9495667629415685, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516200621393698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0132638159623237, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07754473186331245, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1192, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9229571984435798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9229571984435798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9229571984435798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276264591439689} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_46000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad634ef9890e79c383656c9170f8002c803f5daa --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.179268523782452, "train": {"Classification_binary": 2.458129405975342, "Classification": 0.018435444682836533, "Regression_pt": 0.12116580456495285, "Regression_eta": 0.010196195915341377, "Regression_sin_phi": 0.009112844243645668, "Regression_cos_phi": 0.00861384067684412, "Regression_energy": 0.12182242423295975, "Total": 8.113332748413086, "Total_unweighted": 2.747475960291922}, "train_group": {}, "valid": {"Classification_binary": 2.4207396507263184, "Classification": 0.01897879069050153, "Regression_pt": 0.12649158636728922, "Regression_eta": 0.010358693078160286, "Regression_sin_phi": 0.007981139545639357, "Regression_cos_phi": 0.008160756280024847, "Regression_energy": 0.1293458640575409, "Total": 8.166150410970053, "Total_unweighted": 2.7220564807454743, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019665636122226715, "diagnostic/loss/clic_pf/Classification_binary": 2.431057929992676, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007714971899986267, "diagnostic/loss/clic_pf/Regression_energy": 0.13146930932998657, "diagnostic/loss/clic_pf/Regression_eta": 0.009678546339273453, "diagnostic/loss/clic_pf/Regression_pt": 0.12866437435150146, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007641964126378298, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03033151850104332, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1802881509065628, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04588514193892479, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4116900861263275, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.012619762681424618, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1776176542043686, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0562247708439827, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4159206449985504, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9686524868011475, "metrics/particle/class_1/pid_accuracy": 0.9870510697364807, "metrics/particle/class_2/efficiency": 0.45038339495658875, "metrics/particle/class_2/pid_accuracy": 0.7874311208724976, "metrics/particle/class_3/efficiency": 0.7625606060028076, "metrics/particle/class_3/pid_accuracy": 0.9865456819534302, "metrics/particle/class_4/efficiency": 0.9234496355056763, "metrics/particle/class_4/pid_accuracy": 0.4858342111110687, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7221095561981201, "metrics/particle/count/bias_mean": -0.2862745225429535, "metrics/particle/count/mae": 3.0614378452301025, "metrics/particle/count/prediction_mean": 69.71307373046875, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04617252200841904, "metrics/particle/event/energy_response_mean": 1.0061264038085938, "metrics/particle/event/met_abs_error": 4.211167812347412, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03535275533795357, "metrics/particle/event/scalar_pt_response_mean": 1.0070146322250366, "metrics/particle/event/vector_pt_closure": 0.037711918354034424, "metrics/particle/matched/delta_eta_abs_mean": 0.007635691203176975, "metrics/particle/matched/delta_phi_abs_mean": 0.009252961724996567, "metrics/particle/matched/delta_r_mean": 0.013482705689966679, "metrics/particle/matched/energy_relative_abs_mean": 0.08003643155097961, "metrics/particle/matched/pid_accuracy": 0.9711554646492004, "metrics/particle/matched/pt_relative_abs_mean": 0.07390416413545609, "metrics/particle/matching/duplicate_fraction": 0.026870178058743477, "metrics/particle/matching/efficiency": 0.8361329436302185, "metrics/particle/matching/f1": 0.8378461599349976, "metrics/particle/matching/purity": 0.8395664691925049, "metrics/particle/matching/target_coverage_dr0p05": 0.8163941502571106, "metrics/particle/matching/target_coverage_dr0p10": 0.8662172555923462, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7457305788993835, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8735749125480652, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015853190794587135, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07712053507566452, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07302597910165787, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009030699729919434, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1282079517841339, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002874554367735982, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31390661001205444, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.010946346446871758, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.164038896560669, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8697.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27247223258018494, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8737216591835022, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9613253474235535, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.844878077507019, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1160876601934433, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 17.57201385498047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012728490866720676, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7001845836639404, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03909366950392723, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5988216400146484, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10449.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8090985417366028, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8189520239830017, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.544496536254883, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.462925672531128, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.12639057636260986, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.13290715217590332, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.776483535766602, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.23718124628067017, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.461116790771484, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024113614112138748, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08868033438920975, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08670297265052795, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.004888082854449749, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14250604808330536, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019881784915924072, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9290747046470642, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.03671536222100258, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.3761186599731445, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1861.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7332329750061035, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2574132680892944, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11510.328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11510.1533203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2465277761220932, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 617537.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07500000298023224, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7073278427124023, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.15524333715438843, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.287610054016113, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2880.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6057942509651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1588475704193115, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.5622477531433105, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0232279300689697, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0471106581389904, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.01887926459312439, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.4359450340271, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.04929739981889725, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.906362533569336, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011547566391527653, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08098392188549042, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0735890343785286, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010625213384628296, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1298547238111496, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007244127336889505, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1089681014418602, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00026665491168387234, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5957842469215393, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19326.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1346752941608429, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6015191078186035, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4533649981021881, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3526116609573364, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08017580211162567, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.478930950164795, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002174415858462453, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21888472139835358, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0025209318846464157, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0908993482589722, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21615.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8802951574325562, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.9246407747268677, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.397487640380859, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.526429176330566, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.06548679620027542, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0182559490203857, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.510638475418091, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.10421597212553024, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.647825241088867, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013583150692284107, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08033886551856995, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07424183934926987, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009803895838558674, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13020427525043488, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025431669782847166, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21968182921409607, "metrics/particle/momentum/all/accepted/pz_bias": -0.001071692444384098, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1548588275909424, "metrics/particle/momentum/all/accepted/samples": 29884.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22521118819713593, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7369706630706787, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 949.2213745117188, "metrics/particle/momentum/all/assigned/pt_relative_bias": 949.10986328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10462453961372375, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 177285.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011332417838275433, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5678951144218445, "metrics/particle/momentum/all/assigned/pz_bias": 0.022925270721316338, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.901118040084839, "metrics/particle/momentum/all/assigned/samples": 34944.0, "metrics/particle/momentum/all/efficiency": 0.835331916809082, "metrics/particle/momentum/all/event/energy_bias": -1.0531587600708008, "metrics/particle/momentum/all/event/p3_error_mean": 12.203714370727539, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.835382461547852, "metrics/particle/momentum/all/event/px_bias": 0.2389880120754242, "metrics/particle/momentum/all/event/py_bias": -0.8664696216583252, "metrics/particle/momentum/all/event/pz_abs_mean": 9.257519721984863, "metrics/particle/momentum/all/event/pz_bias": 0.39069464802742004, "metrics/particle/momentum/all/event/pz_residual_rms": 16.189237594604492, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007343262899667025, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.031564876437187195, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021130232140421867, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0027748767752200365, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06120825186371803, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001532802009023726, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10958382487297058, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.003918745089322329, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8496261239051819, "metrics/particle/momentum/class_1/accepted/samples": 16310.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.043226953595876694, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1080334484577179, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10871728509664536, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07821278274059296, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02020980231463909, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.837547779083252, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004326438531279564, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2207132875919342, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.015042153187096119, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.021151065826416, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9666330814361572, "metrics/particle/momentum/class_1/event/energy_bias": 1.7318060398101807, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.560994148254395, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.510054111480713, "metrics/particle/momentum/class_1/event/px_bias": -0.3421645164489746, "metrics/particle/momentum/class_1/event/py_bias": -0.5517221689224243, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.315223693847656, "metrics/particle/momentum/class_1/event/pz_bias": 0.6145228743553162, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.042431831359863, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035630788654088974, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19538351893424988, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1957242488861084, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02905873768031597, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23459182679653168, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02921348251402378, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.570388913154602, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0414196215569973, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.286008834838867, "metrics/particle/momentum/class_2/accepted/samples": 1335.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8731855750083923, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6739445328712463, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12352.8515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12352.369140625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3473723530769348, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 639807.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07528885453939438, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0098142623901367, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1925569772720337, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.2618513107299805, "metrics/particle/momentum/class_2/assigned/samples": 2683.0, "metrics/particle/momentum/class_2/efficiency": 0.45315682888031006, "metrics/particle/momentum/class_2/event/energy_bias": -6.371525287628174, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.32898998260498, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.368851661682129, "metrics/particle/momentum/class_2/event/px_bias": 0.1918679177761078, "metrics/particle/momentum/class_2/event/py_bias": -0.5186368227005005, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.020282745361328, "metrics/particle/momentum/class_2/event/pz_bias": 0.006624430418014526, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.092400550842285, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01995311677455902, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13543294370174408, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13568155467510223, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024258721619844437, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.177138090133667, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010231923079118133, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21864533424377441, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0012419286649674177, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0664358139038086, "metrics/particle/momentum/class_3/accepted/samples": 11728.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31991666555404663, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4332551956176758, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6405887603759766, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5016175508499146, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1587173342704773, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.011371612548828, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008084074594080448, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.526577889919281, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008737650699913502, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.332163095474243, "metrics/particle/momentum/class_3/assigned/samples": 14844.0, "metrics/particle/momentum/class_3/efficiency": 0.7609654664993286, "metrics/particle/momentum/class_3/event/energy_bias": 4.375395774841309, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.84083366394043, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.639594078063965, "metrics/particle/momentum/class_3/event/px_bias": 0.2615879476070404, "metrics/particle/momentum/class_3/event/py_bias": 0.10667078197002411, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.531277656555176, "metrics/particle/momentum/class_3/event/pz_bias": -0.15258647501468658, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.44190788269043, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0109151815995574, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10042640566825867, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0561346672475338, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0030904861632734537, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10579640418291092, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26079171895980835, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03720548003911972, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8578314781188965, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.12977899610996246, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.322263717651367, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.929369330406189, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.8704538345336914, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 29.222166061401367, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.387722909450531, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.07676813006401062, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.133531093597412, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.5170844197273254, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6021144390106201, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0130919218063354, "metrics/particle/momentum/class_4/event/px_bias": 0.10976952314376831, "metrics/particle/momentum/class_4/event/py_bias": 0.002882756292819977, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0505573749542236, "metrics/particle/momentum/class_4/event/pz_bias": 0.1447702944278717, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.922053813934326, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005271304398775101, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019028618931770325, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015453298576176167, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005502036772668362, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.042833492159843445, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1689552515745163, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.07691702991724014, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6885386109352112, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.010066576302051544, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03165650740265846, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030854733660817146, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006011204328387976, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11658414453268051, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.48474088311195374, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3914702832698822, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.699265718460083, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.2717512249946594, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9351202249526978, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6579582691192627, "metrics/particle/momentum/class_5/event/px_bias": 0.0179271399974823, "metrics/particle/momentum/class_5/event/py_bias": 0.0943358838558197, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5490607619285583, "metrics/particle/momentum/class_5/event/pz_bias": -0.22263652086257935, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.29862117767334, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4884307384490967, "step_valid_time": 7383.185549497604, "step_total_time": 7384.673981666565, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005570838252216, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05799741304513317, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2890, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2882, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585406301824212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547406673273868, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566368752068851, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045259332672613145, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9558872305140962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520977865873802, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.953988745448527, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585406301824212, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997680659789155, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06917973898599916, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2140, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9498446515756769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545049063336307, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521690767519466, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045495093666369314, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527207850133809, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503893214682981, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9498446515756769, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011313545483267, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07479262597505154, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1192, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1186, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9297971918876755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9287105570705103, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07020280811232449, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9229571984435798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9251170046801872, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9240358395013635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276264591439689} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_47000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..56abd05703f697242ad7beabee882e8731856d5d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226643926473375, "train": {"Classification_binary": 2.3946728706359863, "Classification": 0.01975969225168228, "Regression_pt": 0.12341469526290894, "Regression_eta": 0.008430029265582561, "Regression_sin_phi": 0.006797208450734615, "Regression_cos_phi": 0.005497104488313198, "Regression_energy": 0.12377944588661194, "Total": 7.727230072021484, "Total_unweighted": 2.68235104624182}, "train_group": {}, "valid": {"Classification_binary": 2.416593869527181, "Classification": 0.01887868468960126, "Regression_pt": 0.1258849004904429, "Regression_eta": 0.010278540352980295, "Regression_sin_phi": 0.007929683973391851, "Regression_cos_phi": 0.00808115986486276, "Regression_energy": 0.12867737809816995, "Total": 8.128971099853516, "Total_unweighted": 2.71632421699663, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01952325738966465, "diagnostic/loss/clic_pf/Classification_binary": 2.424912691116333, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007625125348567963, "diagnostic/loss/clic_pf/Regression_energy": 0.1306757777929306, "diagnostic/loss/clic_pf/Regression_eta": 0.00956666935235262, "diagnostic/loss/clic_pf/Regression_pt": 0.1279955804347992, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007629473228007555, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03566765785217285, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1813131719827652, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05122128129005432, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4131828546524048, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.008440539240837097, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17822062969207764, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06040399894118309, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41759586334228516, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.967877209186554, "metrics/particle/class_1/pid_accuracy": 0.9878417253494263, "metrics/particle/class_2/efficiency": 0.4492557644844055, "metrics/particle/class_2/pid_accuracy": 0.7816265225410461, "metrics/particle/class_3/efficiency": 0.7646082639694214, "metrics/particle/class_3/pid_accuracy": 0.9875401854515076, "metrics/particle/class_4/efficiency": 0.9244186282157898, "metrics/particle/class_4/pid_accuracy": 0.472746342420578, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.722449004650116, "metrics/particle/count/bias_mean": -0.20980392396450043, "metrics/particle/count/mae": 3.0006535053253174, "metrics/particle/count/prediction_mean": 69.78954315185547, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04692520946264267, "metrics/particle/event/energy_response_mean": 1.0053207874298096, "metrics/particle/event/met_abs_error": 4.1604084968566895, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03622188791632652, "metrics/particle/event/scalar_pt_response_mean": 1.0070570707321167, "metrics/particle/event/vector_pt_closure": 0.038252998143434525, "metrics/particle/matched/delta_eta_abs_mean": 0.007439992390573025, "metrics/particle/matched/delta_phi_abs_mean": 0.00897657498717308, "metrics/particle/matched/delta_r_mean": 0.013102245517075062, "metrics/particle/matched/energy_relative_abs_mean": 0.0802014172077179, "metrics/particle/matched/pid_accuracy": 0.9716160893440247, "metrics/particle/matched/pt_relative_abs_mean": 0.073612280189991, "metrics/particle/matching/duplicate_fraction": 0.026812639087438583, "metrics/particle/matching/efficiency": 0.8365437388420105, "metrics/particle/matching/f1": 0.8377993106842041, "metrics/particle/matching/purity": 0.8390585780143738, "metrics/particle/matching/target_coverage_dr0p05": 0.8182989358901978, "metrics/particle/matching/target_coverage_dr0p10": 0.8675897717475891, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7467203140258789, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8736869692802429, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015240269713103771, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07731353491544724, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07289905846118927, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01020596083253622, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12743541598320007, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028738935943692923, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3111160397529602, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0010538167553022504, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1545324325561523, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8699.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2644529342651367, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7623931765556335, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9076747894287109, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7896648049354553, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11696633696556091, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 16.011531829833984, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011954858899116516, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6876948475837708, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.003743657609447837, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5308711528778076, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10456.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8092845678329468, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9981479048728943, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.8139519691467285, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6172921657562256, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.08850438892841339, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07739926129579544, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.984223365783691, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.30444997549057007, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.01180648803711, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02396515943109989, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08922156691551208, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08698046207427979, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012454434297978878, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14336512982845306, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01982850953936577, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9173086285591125, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.006438644137233496, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2455685138702393, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1866.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7201091647148132, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4104914665222168, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3520.1904296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3520.0322265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24261383712291718, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 188640.921875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07090719789266586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7602603435516357, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1127137616276741, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.49887466430664, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2877.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.607421875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.3866453170776367, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.576028823852539, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0047897100448608, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05520816147327423, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09835313260555267, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.454376697540283, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.14054247736930847, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.803057670593262, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01119063887745142, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08057039976119995, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07251019030809402, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01112404465675354, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1280929148197174, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006216972251422703, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10442224889993668, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0030904191080480814, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5742847323417664, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19302.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13601422309875488, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5756335258483887, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4279301166534424, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3282819390296936, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08093608170747757, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.047515392303467, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026362037751823664, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22438675165176392, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0014118565013632178, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1690680980682373, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21622.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8792019486427307, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.8412163257598877, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.338657855987549, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.510302543640137, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.09112197905778885, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.13500872254371643, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.4346654415130615, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.024833910167217255, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.874115943908691, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013168239034712315, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08016230911016464, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0735275000333786, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01093976479023695, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12891033291816711, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002477650996297598, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21541012823581696, "metrics/particle/momentum/all/accepted/pz_bias": 0.0019018949242308736, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1222680807113647, "metrics/particle/momentum/all/accepted/samples": 29867.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22250817716121674, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7002121210098267, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 290.2684020996094, "metrics/particle/momentum/all/assigned/pt_relative_bias": 290.1584167480469, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10502073913812637, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 54119.17578125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011042769066989422, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5716920495033264, "metrics/particle/momentum/all/assigned/pz_bias": 0.011270157992839813, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1918258666992188, "metrics/particle/momentum/all/assigned/samples": 34955.0, "metrics/particle/momentum/all/efficiency": 0.8348567485809326, "metrics/particle/momentum/all/event/energy_bias": -1.543576955795288, "metrics/particle/momentum/all/event/p3_error_mean": 12.56167984008789, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.905588150024414, "metrics/particle/momentum/all/event/px_bias": -0.05259054899215698, "metrics/particle/momentum/all/event/py_bias": 0.31076109409332275, "metrics/particle/momentum/all/event/pz_abs_mean": 9.58183479309082, "metrics/particle/momentum/all/event/pz_bias": -0.4201584756374359, "metrics/particle/momentum/all/event/pz_residual_rms": 16.954383850097656, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0068941726349294186, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03232061490416527, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02091660536825657, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004278963897377253, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05985134094953537, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011670761741697788, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10476881265640259, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0022055429872125387, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8090004920959473, "metrics/particle/momentum/class_1/accepted/samples": 16280.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04402158781886101, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1117262989282608, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11607499420642853, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08683779835700989, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021276595070958138, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.39422607421875, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004326438531279564, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2279175966978073, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.009668629616498947, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.076005697250366, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9648550748825073, "metrics/particle/momentum/class_1/event/energy_bias": 3.023068428039551, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.876459121704102, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.758393287658691, "metrics/particle/momentum/class_1/event/px_bias": -0.5220129489898682, "metrics/particle/momentum/class_1/event/py_bias": -0.2862168550491333, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.5249834060668945, "metrics/particle/momentum/class_1/event/pz_bias": 0.36780792474746704, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.204995155334473, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034815263003110886, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1958373636007309, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19550196826457977, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.039849087595939636, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23491431772708893, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03247734159231186, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5846645832061768, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.054719243198633194, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3374767303466797, "metrics/particle/momentum/class_2/accepted/samples": 1324.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8801336288452148, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6667454242706299, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 3745.508544921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 3745.019287109375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3567727506160736, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 194653.921875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07475943863391876, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9989235401153564, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08316103368997574, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.322946071624756, "metrics/particle/momentum/class_2/assigned/samples": 2702.0, "metrics/particle/momentum/class_2/efficiency": 0.4494229555130005, "metrics/particle/momentum/class_2/event/energy_bias": -8.74140739440918, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.474132537841797, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.265259742736816, "metrics/particle/momentum/class_2/event/px_bias": 0.10199445486068726, "metrics/particle/momentum/class_2/event/py_bias": -0.13586607575416565, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.31617546081543, "metrics/particle/momentum/class_2/event/pz_bias": -0.9040523171424866, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.616922378540039, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019612476229667664, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13360154628753662, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13406208157539368, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02615802176296711, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1752762794494629, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001020842231810093, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21440500020980835, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.009366518817842007, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0062086582183838, "metrics/particle/momentum/class_3/accepted/samples": 11755.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3109931945800781, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3554304838180542, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5770459175109863, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4420396089553833, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15630897879600525, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.398832321166992, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007279590237885714, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5251339673995972, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.008450341410934925, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.0129613876342773, "metrics/particle/momentum/class_3/assigned/samples": 14836.0, "metrics/particle/momentum/class_3/efficiency": 0.7627173662185669, "metrics/particle/momentum/class_3/event/energy_bias": 4.812300682067871, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.971665382385254, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.658218860626221, "metrics/particle/momentum/class_3/event/px_bias": 0.24731093645095825, "metrics/particle/momentum/class_3/event/py_bias": 0.6635698676109314, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.664192199707031, "metrics/particle/momentum/class_3/event/pz_bias": 0.16881994903087616, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.578296661376953, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010754468850791454, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10532576590776443, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.054861489683389664, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004535965155810118, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1027088612318039, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.25261402130126953, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.022451315075159073, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8317094445228577, "metrics/particle/momentum/class_4/accepted/samples": 331.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10924595594406128, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8576968908309937, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6735230684280396, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.609878420829773, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.916259765625, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3804759681224823, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.08226551860570908, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1057711839675903, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9143646359443665, "metrics/particle/momentum/class_4/event/energy_bias": -0.42870184779167175, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5984959602355957, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0235902070999146, "metrics/particle/momentum/class_4/event/px_bias": 0.1349988430738449, "metrics/particle/momentum/class_4/event/py_bias": -0.009274763986468315, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0381889343261719, "metrics/particle/momentum/class_4/event/pz_bias": 0.1897389143705368, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.9519124031066895, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004852515645325184, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019154302775859833, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014803587459027767, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004788988269865513, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0423409678041935, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14675894379615784, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0526898168027401, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6889004707336426, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.018865587189793587, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.041522037237882614, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03717108443379402, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.016630783677101135, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14459683001041412, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5831367373466492, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4916519224643707, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.305970191955566, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.208836168050766, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8968826532363892, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6291821002960205, "metrics/particle/momentum/class_5/event/px_bias": -0.014881841838359833, "metrics/particle/momentum/class_5/event/py_bias": 0.07854893803596497, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5289060473442078, "metrics/particle/momentum/class_5/event/pz_bias": -0.2424730658531189, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.297119379043579, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4235608577728271, "step_valid_time": 7524.74610042572, "step_total_time": 7526.169661521912, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000766091569047, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05749118180510926, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540799471423852, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9559748427672956, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0459200528576148, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9562189054726368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9524281466798811, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9543197616683218, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578772802653399, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002604956945345, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07022170072448652, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9537366548042705, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952677182848256, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046263345195729534, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9498446515756769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519572953736655, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9508998000444345, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516200621393698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042801474235774, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07508080817039497, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9245136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9245136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9245136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0754863813229572, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9245136186770428} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_48000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..114407ee9b277053f49f036c1df25562b3d221e6 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.274019329164298, "train": {"Classification_binary": 2.4356184005737305, "Classification": 0.019280076026916504, "Regression_pt": 0.1154429242014885, "Regression_eta": 0.008297820575535297, "Regression_sin_phi": 0.007739280816167593, "Regression_cos_phi": 0.008289198391139507, "Regression_energy": 0.11671674251556396, "Total": 7.761122703552246, "Total_unweighted": 2.711384443100542}, "train_group": {}, "valid": {"Classification_binary": 2.413196245829264, "Classification": 0.018841851502656937, "Regression_pt": 0.12582213679949442, "Regression_eta": 0.01035184289018313, "Regression_sin_phi": 0.008043413360913595, "Regression_cos_phi": 0.008147762467463812, "Regression_energy": 0.12860322992006937, "Total": 8.138102849324545, "Total_unweighted": 2.7130064827700453, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019419968128204346, "diagnostic/loss/clic_pf/Classification_binary": 2.422945499420166, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00761264655739069, "diagnostic/loss/clic_pf/Regression_energy": 0.13016709685325623, "diagnostic/loss/clic_pf/Regression_eta": 0.00959920883178711, "diagnostic/loss/clic_pf/Regression_pt": 0.12750574946403503, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007641736418008804, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04057043418288231, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18155619502067566, "diagnostic/regression/clic_pf/energy_residual_mean": 0.056124065071344376, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4114851653575897, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005027450155466795, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17829909920692444, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0638170838356018, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41534045338630676, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9668038487434387, "metrics/particle/class_1/pid_accuracy": 0.9873758554458618, "metrics/particle/class_2/efficiency": 0.4555705785751343, "metrics/particle/class_2/pid_accuracy": 0.7804455161094666, "metrics/particle/class_3/efficiency": 0.7638969421386719, "metrics/particle/class_3/pid_accuracy": 0.9867667555809021, "metrics/particle/class_4/efficiency": 0.9195736646652222, "metrics/particle/class_4/pid_accuracy": 0.4826132655143738, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7174796462059021, "metrics/particle/count/bias_mean": -0.26143792271614075, "metrics/particle/count/mae": 3.024836540222168, "metrics/particle/count/prediction_mean": 69.73790740966797, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04792582243680954, "metrics/particle/event/energy_response_mean": 1.0142592191696167, "metrics/particle/event/met_abs_error": 4.230340480804443, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037090253084897995, "metrics/particle/event/scalar_pt_response_mean": 1.0121185779571533, "metrics/particle/event/vector_pt_closure": 0.03817036375403404, "metrics/particle/matched/delta_eta_abs_mean": 0.007976851426064968, "metrics/particle/matched/delta_phi_abs_mean": 0.009127089753746986, "metrics/particle/matched/delta_r_mean": 0.013644134625792503, "metrics/particle/matched/energy_relative_abs_mean": 0.0818578228354454, "metrics/particle/matched/pid_accuracy": 0.9709688425064087, "metrics/particle/matched/pt_relative_abs_mean": 0.0749727264046669, "metrics/particle/matching/duplicate_fraction": 0.02707616798579693, "metrics/particle/matching/efficiency": 0.8362262845039368, "metrics/particle/matching/f1": 0.8377907872200012, "metrics/particle/matching/purity": 0.8393611907958984, "metrics/particle/matching/target_coverage_dr0p05": 0.8171224594116211, "metrics/particle/matching/target_coverage_dr0p10": 0.8675897717475891, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7438631653785706, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8727905750274658, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016313452273607254, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07979430258274078, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0746096670627594, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016212530434131622, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.129767507314682, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002998500829562545, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3203987777233124, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.007094433065503836, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1710797548294067, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8671.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2708958089351654, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8775652050971985, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0371665954589844, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9222756028175354, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.118796706199646, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.86380958557129, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012454492971301079, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7105419635772705, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03315172344446182, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.592602252960205, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10438.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8066796660423279, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.21519047021865845, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.428801536560059, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.457664728164673, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.23700085282325745, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2912804186344147, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.647455215454102, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.049758657813072205, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.355306625366211, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024606380611658096, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09281820803880692, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08953922986984253, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01963144540786743, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1463744342327118, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020419130101799965, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9133774638175964, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06804769486188889, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2068376541137695, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1861.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6931450963020325, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2932361364364624, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8784.4111328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8784.255859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24247725307941437, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 469444.9375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06892932206392288, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7072064876556396, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.043432675302028656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.769552230834961, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2858.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6057942509651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.34446144104003906, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.468273162841797, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.036818027496338, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04444380849599838, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1852693259716034, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.335683822631836, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6356837749481201, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.534440994262695, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011479856446385384, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08152006566524506, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07359869033098221, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016477394849061966, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12886883318424225, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006210858700796962, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10603953152894974, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006912982556968927, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5611655116081238, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19321.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13535039126873016, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5755634903907776, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.45085781812667847, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3563045561313629, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0796816423535347, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.803034782409668, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0021748184226453304, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21843931078910828, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006942138075828552, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1078612804412842, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21611.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8800674080848694, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0588023662567139, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.367025852203369, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.519839763641357, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4482969045639038, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.042912598699331284, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.465024709701538, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2179272174835205, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.521450042724609, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013702097348868847, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08172311633825302, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07488604635000229, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01659708097577095, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13028834760189056, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002545807743445039, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21863004565238953, "metrics/particle/momentum/all/accepted/pz_bias": -0.010776749812066555, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1150020360946655, "metrics/particle/momentum/all/accepted/samples": 29853.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22155088186264038, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7246281504631042, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 719.8102416992188, "metrics/particle/momentum/all/assigned/pt_relative_bias": 719.7047729492188, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1047067940235138, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 134325.828125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010714183561503887, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5693564414978027, "metrics/particle/momentum/all/assigned/pz_bias": 0.0020592017099261284, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0110373497009277, "metrics/particle/momentum/all/assigned/samples": 34907.0, "metrics/particle/momentum/all/efficiency": 0.8344653844833374, "metrics/particle/momentum/all/event/energy_bias": 0.9295315742492676, "metrics/particle/momentum/all/event/p3_error_mean": 12.406000137329102, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.967036724090576, "metrics/particle/momentum/all/event/px_bias": 0.7297415733337402, "metrics/particle/momentum/all/event/py_bias": 0.5194623470306396, "metrics/particle/momentum/all/event/pz_abs_mean": 9.358627319335938, "metrics/particle/momentum/all/event/pz_bias": -0.3679979145526886, "metrics/particle/momentum/all/event/pz_residual_rms": 16.598995208740234, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007085676304996014, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03334725648164749, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022061433643102646, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007156079635024071, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06178312003612518, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016583747928962111, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11215664446353912, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01085422933101654, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.852329671382904, "metrics/particle/momentum/class_1/accepted/samples": 16281.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04487232863903046, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11243496090173721, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11310867965221405, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08536899834871292, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021752014756202698, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.037057399749756, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004267425276339054, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22410431504249573, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.002663385821506381, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0219995975494385, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9649143815040588, "metrics/particle/momentum/class_1/event/energy_bias": 3.4102845191955566, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.956506729125977, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.698930740356445, "metrics/particle/momentum/class_1/event/px_bias": -0.45696502923965454, "metrics/particle/momentum/class_1/event/py_bias": -0.3663876950740814, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.637313365936279, "metrics/particle/momentum/class_1/event/pz_bias": -0.01078614592552185, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.370357513427734, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03606355935335159, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1930396407842636, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19416679441928864, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03451318293809891, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23346328735351562, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028549961745738983, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.545559287071228, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11486850678920746, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.25594425201416, "metrics/particle/momentum/class_2/accepted/samples": 1331.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8692037463188171, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6120470762252808, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 9306.072265625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 9305.5947265625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3448275923728943, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 483253.84375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06933630257844925, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9114058017730713, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.050463877618312836, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.985453128814697, "metrics/particle/momentum/class_2/assigned/samples": 2697.0, "metrics/particle/momentum/class_2/efficiency": 0.45179903507232666, "metrics/particle/momentum/class_2/event/energy_bias": -7.509017467498779, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.577942848205566, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.490171432495117, "metrics/particle/momentum/class_2/event/px_bias": 0.4122530221939087, "metrics/particle/momentum/class_2/event/py_bias": -0.24353133141994476, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.32024097442627, "metrics/particle/momentum/class_2/event/pz_bias": -0.3360903859138489, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.301575660705566, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02054055966436863, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13653162121772766, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13605089485645294, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03596259281039238, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17716504633426666, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009376065572723746, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21519164741039276, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.00265755713917315, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9592791199684143, "metrics/particle/momentum/class_3/accepted/samples": 11732.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3098297715187073, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.417181134223938, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7101582288742065, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5815339088439941, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15756388008594513, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.887300491333008, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007638231851160526, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5410772562026978, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0030534514226019382, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7620294094085693, "metrics/particle/momentum/class_3/assigned/samples": 14794.0, "metrics/particle/momentum/class_3/efficiency": 0.7612250447273254, "metrics/particle/momentum/class_3/event/energy_bias": 5.758157253265381, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.30441951751709, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.8867106437683105, "metrics/particle/momentum/class_3/event/px_bias": 0.644497811794281, "metrics/particle/momentum/class_3/event/py_bias": 1.0336328744888306, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.921931266784668, "metrics/particle/momentum/class_3/event/pz_bias": 0.06574512273073196, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.261739730834961, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011393936350941658, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10261103510856628, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05583646893501282, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004024164285510778, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10015083104372025, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2500479817390442, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.011157865636050701, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8294520974159241, "metrics/particle/momentum/class_4/accepted/samples": 331.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.12451784312725067, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1366686820983887, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0000815391540527, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9394474029541016, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0635359138250351, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.682537078857422, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38340699672698975, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.030497213825583458, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.108146071434021, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9143646359443665, "metrics/particle/momentum/class_4/event/energy_bias": -0.5485504865646362, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5690771341323853, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9912355542182922, "metrics/particle/momentum/class_4/event/px_bias": 0.10832644253969193, "metrics/particle/momentum/class_4/event/py_bias": -0.012220797128975391, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.029611349105835, "metrics/particle/momentum/class_4/event/pz_bias": 0.17334480583667755, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.930677890777588, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00524156866595149, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022835271432995796, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01867538131773472, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009305956773459911, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05328112468123436, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2034088373184204, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.11008886992931366, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9261336326599121, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.020060673356056213, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04204171523451805, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03785969316959381, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.009247877635061741, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.14242705702781677, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5378298759460449, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29090332984924316, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7713897228240967, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.18134211003780365, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9764890670776367, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6804031133651733, "metrics/particle/momentum/class_5/event/px_bias": 0.02162931114435196, "metrics/particle/momentum/class_5/event/py_bias": 0.1079692468047142, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.580552339553833, "metrics/particle/momentum/class_5/event/pz_bias": -0.2602112293243408, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3495547771453857, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.434694528579712, "step_valid_time": 7665.695784807205, "step_total_time": 7667.1304795742035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004060433117568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05810330372534733, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.96318407960199, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956836902800659, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.96, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04316309719934102, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9611940298507463, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548599670510708, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9580165289256198, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.96318407960199, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004514514364631, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07171631227000275, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529516200621394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542222222222222, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535864978902954, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04577777777777778, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9511762094984465, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9524444444444444, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9518099045081057, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529516200621394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0053505733535895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07187077680113818, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9235569422776911, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9224776003116478, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07644305772230889, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9198443579766536, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.921996879875195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9209193611219322, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9214007782101167} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_49000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..c54dddd6023e012b0207cd588ac9de83ca6dc726 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.3213947318552206, "train": {"Classification_binary": 2.318869113922119, "Classification": 0.017203571274876595, "Regression_pt": 0.12229309976100922, "Regression_eta": 0.008024199865758419, "Regression_sin_phi": 0.006024921778589487, "Regression_cos_phi": 0.006621653214097023, "Regression_energy": 0.12330124527215958, "Total": 7.524824142456055, "Total_unweighted": 2.6023378050886095}, "train_group": {}, "valid": {"Classification_binary": 2.4091968536376953, "Classification": 0.01882286990682284, "Regression_pt": 0.12548110882441202, "Regression_eta": 0.010289442415038744, "Regression_sin_phi": 0.007885605717698732, "Regression_cos_phi": 0.008031919598579407, "Regression_energy": 0.1280121405919393, "Total": 8.099107106526693, "Total_unweighted": 2.7077199406921864, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01941109262406826, "diagnostic/loss/clic_pf/Classification_binary": 2.4167544841766357, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007588043808937073, "diagnostic/loss/clic_pf/Regression_energy": 0.13027457892894745, "diagnostic/loss/clic_pf/Regression_eta": 0.009589983150362968, "diagnostic/loss/clic_pf/Regression_pt": 0.1280110627412796, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007633490487933159, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04329386726021767, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18251921236515045, "diagnostic/regression/clic_pf/energy_residual_mean": 0.058847490698099136, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41536998748779297, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0007249885238707066, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17986848950386047, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06811955571174622, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42071154713630676, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9681754112243652, "metrics/particle/class_1/pid_accuracy": 0.9874964356422424, "metrics/particle/class_2/efficiency": 0.450045108795166, "metrics/particle/class_2/pid_accuracy": 0.7892758846282959, "metrics/particle/class_3/efficiency": 0.7580558061599731, "metrics/particle/class_3/pid_accuracy": 0.986181378364563, "metrics/particle/class_4/efficiency": 0.9282945990562439, "metrics/particle/class_4/pid_accuracy": 0.5, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7174796462059021, "metrics/particle/count/bias_mean": -0.38235294818878174, "metrics/particle/count/mae": 3.060784339904785, "metrics/particle/count/prediction_mean": 69.61699676513672, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04817141219973564, "metrics/particle/event/energy_response_mean": 1.0141887664794922, "metrics/particle/event/met_abs_error": 4.218664646148682, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036913614720106125, "metrics/particle/event/scalar_pt_response_mean": 1.0121128559112549, "metrics/particle/event/vector_pt_closure": 0.03862573578953743, "metrics/particle/matched/delta_eta_abs_mean": 0.007777062710374594, "metrics/particle/matched/delta_phi_abs_mean": 0.009199801832437515, "metrics/particle/matched/delta_r_mean": 0.013529548421502113, "metrics/particle/matched/energy_relative_abs_mean": 0.08073810487985611, "metrics/particle/matched/pid_accuracy": 0.9714052081108093, "metrics/particle/matched/pt_relative_abs_mean": 0.07416002452373505, "metrics/particle/matching/duplicate_fraction": 0.027526898309588432, "metrics/particle/matching/efficiency": 0.8339667320251465, "metrics/particle/matching/f1": 0.8362506031990051, "metrics/particle/matching/purity": 0.8385470509529114, "metrics/particle/matching/target_coverage_dr0p05": 0.8154417872428894, "metrics/particle/matching/target_coverage_dr0p10": 0.8662359118461609, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7425932884216309, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8707457780838013, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01569889672100544, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07811594754457474, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07351232320070267, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014211197383701801, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12852256000041962, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0027672084979712963, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3155735731124878, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02101035788655281, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1803492307662964, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8673.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27381426095962524, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7588692903518677, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7200961112976074, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6028762459754944, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11928010731935501, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.8338775634765625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012444955296814442, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6983103156089783, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.021681612357497215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5426924228668213, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10446.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8068657517433167, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.22274819016456604, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.589880466461182, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.503506660461426, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.11469429731369019, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.27062103152275085, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.799068450927734, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.85823655128479, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.64378547668457, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024909185245633125, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09282244741916656, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08982259780168533, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021288471296429634, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1467951089143753, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018817204982042313, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9214527010917664, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.008961687795817852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.230397939682007, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1860.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7122195363044739, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2246500253677368, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 69439.46875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 69439.3046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2504335641860962, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3728291.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0738813728094101, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8718345165252686, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03821277245879173, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.639455795288086, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2883.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.60546875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.35100841522216797, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.399688720703125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9718252420425415, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0912313312292099, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1485021710395813, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.283634662628174, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.14128513634204865, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.905147552490234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011670669540762901, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08068773150444031, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07292566448450089, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016292395070195198, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12881682813167572, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006233442691154778, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10589802265167236, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011919383890926838, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5668649077415466, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19251.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.134341761469841, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6419142484664917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4645763635635376, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3712264597415924, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08373859524726868, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.9807000160217285, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024084108881652355, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2255430519580841, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.009852162562310696, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1916909217834473, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21591.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8768789172172546, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3452295064926147, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.324033737182617, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.4344868659973145, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.31292209029197693, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.34889280796051025, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.5322742462158203, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6484239101409912, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.733836650848389, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013670415617525578, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08069664984941483, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07415170967578888, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015998361632227898, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12992753088474274, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023838302586227655, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21788595616817474, "metrics/particle/momentum/all/accepted/pz_bias": -0.013262628577649593, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1247622966766357, "metrics/particle/momentum/all/accepted/samples": 29784.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22377337515354156, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7250109910964966, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 5733.43505859375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 5733.32861328125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10813287645578384, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1071260.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01131156925112009, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5854452848434448, "metrics/particle/momentum/all/assigned/pz_bias": -0.009422587230801582, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2378246784210205, "metrics/particle/momentum/all/assigned/samples": 34920.0, "metrics/particle/momentum/all/efficiency": 0.8325366973876953, "metrics/particle/momentum/all/event/energy_bias": 0.7714728116989136, "metrics/particle/momentum/all/event/p3_error_mean": 12.633478164672852, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.925426483154297, "metrics/particle/momentum/all/event/px_bias": -0.10699646174907684, "metrics/particle/momentum/all/event/py_bias": 0.76801598072052, "metrics/particle/momentum/all/event/pz_abs_mean": 9.717208862304688, "metrics/particle/momentum/all/event/pz_bias": -1.6479456424713135, "metrics/particle/momentum/all/event/pz_residual_rms": 17.13799285888672, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00711293239146471, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.032589565962553024, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021416662260890007, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005129487253725529, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06089409440755844, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013498588232323527, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10926488041877747, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0075295064598321915, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8120107054710388, "metrics/particle/momentum/class_1/accepted/samples": 16298.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04426441714167595, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10054744780063629, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08849111944437027, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.059321340173482895, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020743198692798615, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.723743200302124, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0040301070548594, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22740377485752106, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0002687040832825005, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1682393550872803, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9659218788146973, "metrics/particle/momentum/class_1/event/energy_bias": 2.8811452388763428, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.961654663085938, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.689579010009766, "metrics/particle/momentum/class_1/event/px_bias": -0.5442051887512207, "metrics/particle/momentum/class_1/event/py_bias": -0.3544922471046448, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.6496381759643555, "metrics/particle/momentum/class_1/event/pz_bias": 0.008492976427078247, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.854007720947266, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03540690988302231, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19292593002319336, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19480732083320618, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.032885078340768814, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23355960845947266, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028787879273295403, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5606471300125122, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05984572321176529, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3617420196533203, "metrics/particle/momentum/class_2/accepted/samples": 1320.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8739404678344727, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6425538063049316, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 74336.234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 74335.7421875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3583364188671112, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3857571.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.075009286403656, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9893693923950195, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04306928068399429, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.135590076446533, "metrics/particle/momentum/class_2/assigned/samples": 2693.0, "metrics/particle/momentum/class_2/efficiency": 0.4480651617050171, "metrics/particle/momentum/class_2/event/energy_bias": -6.977286338806152, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.876800537109375, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.665666103363037, "metrics/particle/momentum/class_2/event/px_bias": 0.20046791434288025, "metrics/particle/momentum/class_2/event/py_bias": 0.20597624778747559, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.466078758239746, "metrics/particle/momentum/class_2/event/pz_bias": -0.81443852186203, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.610898971557617, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020585808902978897, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13559846580028534, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1357039511203766, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.037174347788095474, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17734141647815704, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009440439171157777, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21766185760498047, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.014259467832744122, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0006248950958252, "metrics/particle/momentum/class_3/accepted/samples": 11652.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3158447742462158, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4251028299331665, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4759739637374878, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3495203256607056, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16461850702762604, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.33617401123047, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008305199444293976, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5621213912963867, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.024174384772777557, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.145678997039795, "metrics/particle/momentum/class_3/assigned/samples": 14810.0, "metrics/particle/momentum/class_3/efficiency": 0.756034255027771, "metrics/particle/momentum/class_3/event/energy_bias": 5.518648624420166, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.500679969787598, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.993094444274902, "metrics/particle/momentum/class_3/event/px_bias": 0.10624270141124725, "metrics/particle/momentum/class_3/event/py_bias": 0.8363556861877441, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.002265930175781, "metrics/particle/momentum/class_3/event/pz_bias": -0.7823817133903503, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.426278114318848, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010982805863022804, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09904493391513824, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05181489512324333, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.003704673610627651, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09679638594388962, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23502781987190247, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.029800403863191605, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8065442442893982, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09692420810461044, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.147707223892212, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9342937469482422, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.874727725982666, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.028879165649414, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.37776654958724976, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.037213683128356934, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.206952452659607, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.4202820658683777, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.569042444229126, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0011799335479736, "metrics/particle/momentum/class_4/event/px_bias": 0.11918357759714127, "metrics/particle/momentum/class_4/event/py_bias": 0.006443195044994354, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.022518277168274, "metrics/particle/momentum/class_4/event/pz_bias": 0.1503087431192398, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.957403182983398, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005312145221978426, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.025076594203710556, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021004460752010345, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.010651130229234695, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06490418314933777, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.188440203666687, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09590751677751541, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8553423285484314, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.005620154552161694, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03959342837333679, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03556373342871666, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01578701287508011, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13284464180469513, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5199205279350281, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.42891308665275574, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.7537832260131836, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.23075255751609802, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9059202671051025, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6351841688156128, "metrics/particle/momentum/class_5/event/px_bias": 0.0113145112991333, "metrics/particle/momentum/class_5/event/py_bias": 0.07373307645320892, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5371827483177185, "metrics/particle/momentum/class_5/event/pz_bias": -0.2099270522594452, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2848024368286133, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.420060157775879, "step_valid_time": 7807.261909246445, "step_total_time": 7808.681969642639, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0032064606401234, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059055143281323774, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954800395908941, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957327158451869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045199604091059054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.952820851204223, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.955342375124049, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9598673300165838, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031133745684084, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07008163231583464, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9535921463632308, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9510458388963062, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0464078536367693, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9526996876394467, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9501557632398754, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9485130936529073, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060574921865304, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07466758837396559, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1190, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9260700389105059, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9275136399064692, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.926791277258567, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0724863600935308, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9251753702260327, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9244548286604362, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9260700389105059} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_5000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..783bea3eea5cd667b1d082e487e81df2c1f3a24c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23687701345461437, "train": {"Classification_binary": 3.5789618492126465, "Classification": 0.03177535533905029, "Regression_pt": 0.19936621189117432, "Regression_eta": 0.016405826434493065, "Regression_sin_phi": 0.011426773853600025, "Regression_cos_phi": 0.011388882994651794, "Regression_energy": 0.20026570558547974, "Total": 12.481164932250977, "Total_unweighted": 4.049590605311096}, "train_group": {}, "valid": {"Classification_binary": 3.528458913167318, "Classification": 0.03117961436510086, "Regression_pt": 0.18101060390472412, "Regression_eta": 0.014094473173220953, "Regression_sin_phi": 0.011097340534130732, "Regression_cos_phi": 0.01081352929274241, "Regression_energy": 0.18375072876612344, "Total": 11.76541010538737, "Total_unweighted": 3.9604052032033605, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.032173581421375275, "diagnostic/loss/clic_pf/Classification_binary": 3.5227274894714355, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.010492691770195961, "diagnostic/loss/clic_pf/Regression_energy": 0.18403679132461548, "diagnostic/loss/clic_pf/Regression_eta": 0.012951069511473179, "diagnostic/loss/clic_pf/Regression_pt": 0.18162381649017334, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.010984201915562153, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04973220452666283, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.22495533525943756, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0652858316898346, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5043081045150757, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.014222336933016777, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2240774780511856, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08306687325239182, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5107704401016235, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9487149119377136, "metrics/particle/class_1/pid_accuracy": 0.983971357345581, "metrics/particle/class_2/efficiency": 0.3010825514793396, "metrics/particle/class_2/pid_accuracy": 0.5449438095092773, "metrics/particle/class_3/efficiency": 0.6306498646736145, "metrics/particle/class_3/pid_accuracy": 0.9738541841506958, "metrics/particle/class_4/efficiency": 0.9282945990562439, "metrics/particle/class_4/pid_accuracy": 0.17327766120433807, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.45102041959762573, "metrics/particle/count/bias_mean": -1.686928153038025, "metrics/particle/count/mae": 4.212418079376221, "metrics/particle/count/prediction_mean": 68.31241607666016, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.07185860723257065, "metrics/particle/event/energy_response_mean": 0.9994490146636963, "metrics/particle/event/met_abs_error": 8.550348281860352, "metrics/particle/event/scalar_pt_relative_abs_error": 0.058120328933000565, "metrics/particle/event/scalar_pt_response_mean": 1.007542610168457, "metrics/particle/event/vector_pt_closure": 0.06678489595651627, "metrics/particle/matched/delta_eta_abs_mean": 0.01829695515334606, "metrics/particle/matched/delta_phi_abs_mean": 0.01556506659835577, "metrics/particle/matched/delta_r_mean": 0.02669612504541874, "metrics/particle/matched/energy_relative_abs_mean": 0.08431102335453033, "metrics/particle/matched/pid_accuracy": 0.9530726075172424, "metrics/particle/matched/pt_relative_abs_mean": 0.07714731246232986, "metrics/particle/matching/duplicate_fraction": 0.03845270723104477, "metrics/particle/matching/efficiency": 0.7572806477546692, "metrics/particle/matching/f1": 0.7665168642997742, "metrics/particle/matching/purity": 0.7759811878204346, "metrics/particle/matching/target_coverage_dr0p05": 0.6571676731109619, "metrics/particle/matching/target_coverage_dr0p10": 0.7925564050674438, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6819671392440796, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8146294355392456, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02903454750776291, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07876937091350555, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07178543508052826, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00382055900990963, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12789209187030792, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004285522736608982, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3883265256881714, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.06396038085222244, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.413378357887268, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7467.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.45163315534591675, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.4142192602157593, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.6213223934173584, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4775240421295166, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.17609506845474243, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 13.053256034851074, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.021803181618452072, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.1209664344787598, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04064760357141495, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.837637901306152, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10182.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.694669246673584, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.1978259086608887, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.30267333984375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.757050514221191, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.24095866084098816, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.8826590776443481, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.772281646728516, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.17312219738960266, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.16581916809082, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.033106617629528046, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08866236358880997, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0858684629201889, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015051768161356449, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14568786323070526, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.013850415125489235, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8497136235237122, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0158236101269722, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.5220656394958496, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1444.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9029673933982849, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.059023380279541, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.715427875518799, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.5353336334228516, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.32818248867988586, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.795028686523438, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09749816358089447, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.785278797149658, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.07071471959352493, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.158161163330078, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2718.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4700520932674408, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.18610668182373047, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.535798072814941, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.6364564895629883, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.024025682359933853, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0332367978990078, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.340533256530762, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.2012512683868408, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 23.883268356323242, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.025310594588518143, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08601351082324982, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07854504883289337, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01338381227105856, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13550585508346558, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008283174247480929, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14080946147441864, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006686333101242781, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6805533766746521, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18109.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2129206508398056, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.0097230672836304, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7911924719810486, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6736249327659607, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.11560747772455215, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.895149230957031, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004439252428710461, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.36332112550735474, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.013766651973128319, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.7177023887634277, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21400.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8248610496520996, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.5664865970611572, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.813243865966797, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.68980598449707, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.01187063381075859, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.975712537765503, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.889377117156982, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.6450648307800293, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 17.37893295288086, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.026756344363093376, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08415313810110092, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07706840336322784, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010830136947333813, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13401730358600616, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002479644725099206, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2470961958169937, "metrics/particle/momentum/all/accepted/pz_bias": 0.014039918780326843, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0965361595153809, "metrics/particle/momentum/all/accepted/samples": 27020.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3384634256362915, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.2129470109939575, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2693397998809814, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1390308141708374, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15040816366672516, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.72976303100586, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.016967929899692535, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8593918681144714, "metrics/particle/momentum/all/assigned/pz_bias": -0.002126386621966958, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.823781490325928, "metrics/particle/momentum/all/assigned/samples": 34300.0, "metrics/particle/momentum/all/efficiency": 0.7552760243415833, "metrics/particle/momentum/all/event/energy_bias": -0.4452328085899353, "metrics/particle/momentum/all/event/p3_error_mean": 23.259496688842773, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.862306594848633, "metrics/particle/momentum/all/event/px_bias": 0.20506232976913452, "metrics/particle/momentum/all/event/py_bias": -3.825134754180908, "metrics/particle/momentum/all/event/pz_abs_mean": 17.24799156188965, "metrics/particle/momentum/all/event/pz_bias": 0.7293086051940918, "metrics/particle/momentum/all/event/pz_residual_rms": 29.60424041748047, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.015934882685542107, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.036949124187231064, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02609209716320038, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0007359866867773235, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06358102709054947, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011267606168985367, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13222599029541016, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.023395828902721405, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7711885571479797, "metrics/particle/momentum/class_1/accepted/samples": 15975.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07496054470539093, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20660239458084106, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.19834960997104645, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15676796436309814, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03141112998127937, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.20453143119812, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005393231753259897, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3334589898586273, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.052211418747901917, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.691208839416504, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9467788934707642, "metrics/particle/momentum/class_1/event/energy_bias": 5.282931327819824, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.387511253356934, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.153091907501221, "metrics/particle/momentum/class_1/event/px_bias": -0.6491473913192749, "metrics/particle/momentum/class_1/event/py_bias": -0.20694561302661896, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.813175201416016, "metrics/particle/momentum/class_1/event/pz_bias": 1.6620478630065918, "metrics/particle/momentum/class_1/event/pz_residual_rms": 22.871023178100586, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.048106878995895386, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21823711693286896, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2166050672531128, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05461868271231651, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2563132047653198, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03880266100168228, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.889478087425232, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.009617974050343037, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.860962152481079, "metrics/particle/momentum/class_2/accepted/samples": 902.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1722042560577393, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.0469081401824951, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.286868929862976, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.6980838179588318, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.46014493703842163, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.808570384979248, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.10466988384723663, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.051255226135254, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.12494371831417084, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.008938789367676, "metrics/particle/momentum/class_2/assigned/samples": 2484.0, "metrics/particle/momentum/class_2/efficiency": 0.30617785453796387, "metrics/particle/momentum/class_2/event/energy_bias": -12.595672607421875, "metrics/particle/momentum/class_2/event/p3_error_mean": 20.51593017578125, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.519525527954102, "metrics/particle/momentum/class_2/event/px_bias": 0.20700111985206604, "metrics/particle/momentum/class_2/event/py_bias": -2.563969612121582, "metrics/particle/momentum/class_2/event/pz_abs_mean": 15.261510848999023, "metrics/particle/momentum/class_2/event/pz_bias": -1.4439529180526733, "metrics/particle/momentum/class_2/event/pz_residual_rms": 23.843961715698242, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.043225858360528946, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15144973993301392, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.150689497590065, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03462747111916542, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19284175336360931, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014542433200404048, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2817230522632599, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0016441351035609841, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9693512916564941, "metrics/particle/momentum/class_3/accepted/samples": 9627.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.5130211114883423, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.4256656169891357, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.533109188079834, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.37508487701416, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.24043336510658264, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.32008934020996, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.015903882682323456, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9403206706047058, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.041319701820611954, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.681377410888672, "metrics/particle/momentum/class_3/assigned/samples": 14399.0, "metrics/particle/momentum/class_3/efficiency": 0.6246431469917297, "metrics/particle/momentum/class_3/event/energy_bias": 8.879773139953613, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.575387954711914, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.096616268157959, "metrics/particle/momentum/class_3/event/px_bias": 0.34938719868659973, "metrics/particle/momentum/class_3/event/py_bias": -1.2454408407211304, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.045437812805176, "metrics/particle/momentum/class_3/event/pz_bias": -0.04860646277666092, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.260298728942871, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.019326750189065933, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.07006962597370148, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04381662979722023, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.011398992501199245, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.08715280145406723, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26581358909606934, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.06131889298558235, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0004044771194458, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10926292091608047, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.611480712890625, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4193484783172607, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3565646409988403, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.852617263793945, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.352786660194397, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.009351340122520924, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.4650051593780518, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -1.566196322441101, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.6061275005340576, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.7253422737121582, "metrics/particle/momentum/class_4/event/px_bias": 0.24242904782295227, "metrics/particle/momentum/class_4/event/py_bias": 0.13007310032844543, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.644791841506958, "metrics/particle/momentum/class_4/event/pz_bias": 0.5092846751213074, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.010770797729492, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.013161239214241505, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028411610051989555, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026454105973243713, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0034736953675746918, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07053297758102417, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.32509925961494446, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.052789486944675446, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9408006072044373, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033992644399404526, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.038692671805620193, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03834279254078865, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0019196441862732172, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12031065672636032, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6592312455177307, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.28549903631210327, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.092136383056641, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.44606834650039673, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.293023109436035, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.6083052158355713, "metrics/particle/momentum/class_5/event/px_bias": 0.055392347276210785, "metrics/particle/momentum/class_5/event/py_bias": 0.06114814057946205, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.3112375736236572, "metrics/particle/momentum/class_5/event/pz_bias": 0.050535500049591064, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.501045227050781, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2203071117401123, "step_valid_time": 1074.4508090019226, "step_total_time": 1075.6711168289185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003193728500959, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10849458897969921, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2984, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2794, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2769, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9266998341625208, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9363270777479893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9314885814302384, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06367292225201072, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.918407960199005, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9279490616621984, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.923153858976496, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9266998341625208, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.994454387073591, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12875260556479995, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2225, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2041, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2033, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9059032401242787, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9173033707865168, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.911567664135775, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08269662921348314, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.902352418996893, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9137078651685393, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9079946404644931, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9059032401242787, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9970015682249255, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13694057178438124, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1135, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1122, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8832684824902723, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8881064162754304, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8856808427623878, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11189358372456965, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8731517509727627, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8779342723004695, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8755364806866953, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8832684824902723} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_50000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..48675612f6e89a068a56cc3d6d268aed9a8acc64 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3687701345461436, "train": {"Classification_binary": 2.325037717819214, "Classification": 0.015845581889152527, "Regression_pt": 0.12650398910045624, "Regression_eta": 0.007661771960556507, "Regression_sin_phi": 0.007800096180289984, "Regression_cos_phi": 0.007701267022639513, "Regression_energy": 0.1245437040925026, "Total": 7.747241973876953, "Total_unweighted": 2.6150941280648112}, "train_group": {}, "valid": {"Classification_binary": 2.4063207308451333, "Classification": 0.018725210179885227, "Regression_pt": 0.12490421533584595, "Regression_eta": 0.010334582378466925, "Regression_sin_phi": 0.007921909292538961, "Regression_cos_phi": 0.00803235669930776, "Regression_energy": 0.1277755598227183, "Total": 8.086082458496094, "Total_unweighted": 2.7040145645538964, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01931067928671837, "diagnostic/loss/clic_pf/Classification_binary": 2.4122672080993652, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007505784276872873, "diagnostic/loss/clic_pf/Regression_energy": 0.13043852150440216, "diagnostic/loss/clic_pf/Regression_eta": 0.009696432389318943, "diagnostic/loss/clic_pf/Regression_pt": 0.12765063345432281, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007544768042862415, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03864803910255432, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18103204667568207, "diagnostic/regression/clic_pf/energy_residual_mean": 0.054201655089855194, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4119616746902466, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.004969818517565727, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17807409167289734, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06387472152709961, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4169888198375702, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9673603773117065, "metrics/particle/class_1/pid_accuracy": 0.9869927167892456, "metrics/particle/class_2/efficiency": 0.44981956481933594, "metrics/particle/class_2/pid_accuracy": 0.7826523184776306, "metrics/particle/class_3/efficiency": 0.7639400959014893, "metrics/particle/class_3/pid_accuracy": 0.987247109413147, "metrics/particle/class_4/efficiency": 0.9195736646652222, "metrics/particle/class_4/pid_accuracy": 0.4984193742275238, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7128309607505798, "metrics/particle/count/bias_mean": -0.4843137264251709, "metrics/particle/count/mae": 3.0346405506134033, "metrics/particle/count/prediction_mean": 69.51502990722656, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04676876589655876, "metrics/particle/event/energy_response_mean": 1.007863163948059, "metrics/particle/event/met_abs_error": 4.248844146728516, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03691903501749039, "metrics/particle/event/scalar_pt_response_mean": 1.0099446773529053, "metrics/particle/event/vector_pt_closure": 0.038311976939439774, "metrics/particle/matched/delta_eta_abs_mean": 0.007470107637345791, "metrics/particle/matched/delta_phi_abs_mean": 0.008945058099925518, "metrics/particle/matched/delta_r_mean": 0.013099215924739838, "metrics/particle/matched/energy_relative_abs_mean": 0.08083489537239075, "metrics/particle/matched/pid_accuracy": 0.9713079333305359, "metrics/particle/matched/pt_relative_abs_mean": 0.07398535311222076, "metrics/particle/matching/duplicate_fraction": 0.02632618136703968, "metrics/particle/matching/efficiency": 0.8360208868980408, "metrics/particle/matching/f1": 0.8389230370521545, "metrics/particle/matching/purity": 0.8418454527854919, "metrics/particle/matching/target_coverage_dr0p05": 0.8181215524673462, "metrics/particle/matching/target_coverage_dr0p10": 0.8674964308738708, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7456745505332947, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8727065324783325, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01517406851053238, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07819677144289017, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07321806252002716, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.011133012361824512, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12800119817256927, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002757669659331441, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3079044818878174, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.013433008454740047, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1471760272979736, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8703.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2686292827129364, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8728024363517761, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8988169431686401, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7816091775894165, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11766960471868515, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.188153266906738, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013127635233104229, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7129896879196167, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.034544650465250015, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6581146717071533, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10436.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8096567392349243, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.30524224042892456, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.420093059539795, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.4658992290496826, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.0915057361125946, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.05503193289041519, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.649266242980957, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5611007213592529, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.483779907226562, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02387365885078907, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08871235698461533, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08791578561067581, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018320245668292046, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1428854614496231, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01879699155688286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9268726706504822, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06026361510157585, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.333526134490967, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1862.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7032885551452637, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4238460063934326, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 90581.9765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 90581.8203125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24074074625968933, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 4845740.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07092942297458649, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7448372840881348, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10924939811229706, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.386022567749023, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2862.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6061198115348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2591512203216553, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.7907843589782715, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0272350311279297, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07573910802602768, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11440272629261017, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.666234016418457, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.30033189058303833, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.81150245666504, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01115710660815239, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08113624155521393, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07285602390766144, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016423378139734268, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1286291629076004, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005707762320525944, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10269928723573685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.00256716413423419, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.539379894733429, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19272.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12809696793556213, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5588228702545166, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4271102845668793, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3322184383869171, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08110111951828003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.069472312927246, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002131800865754485, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2104269117116928, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.001174749224446714, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0857487916946411, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21578.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8778354525566101, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.7097102403640747, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.213164806365967, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.382065296173096, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.22203829884529114, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3623085021972656, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3838284015655518, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2745034992694855, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.547302722930908, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0131223788484931, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08075164258480072, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07390144467353821, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014998634345829487, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12938322126865387, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023460802622139454, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21398767828941345, "metrics/particle/momentum/all/accepted/pz_bias": 0.0060208444483578205, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1248347759246826, "metrics/particle/momentum/all/accepted/samples": 29837.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21735014021396637, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7237611413002014, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 7433.88623046875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 7433.77978515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1051439419388771, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1388134.75, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011067783460021019, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5687888264656067, "metrics/particle/momentum/all/assigned/pz_bias": 0.018575267866253853, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.1738100051879883, "metrics/particle/momentum/all/assigned/samples": 34876.0, "metrics/particle/momentum/all/efficiency": 0.8340181708335876, "metrics/particle/momentum/all/event/energy_bias": -0.24419879913330078, "metrics/particle/momentum/all/event/p3_error_mean": 12.809347152709961, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.92518424987793, "metrics/particle/momentum/all/event/px_bias": 0.38928312063217163, "metrics/particle/momentum/all/event/py_bias": -0.19287383556365967, "metrics/particle/momentum/all/event/pz_abs_mean": 9.885549545288086, "metrics/particle/momentum/all/event/pz_bias": 1.1359361410140991, "metrics/particle/momentum/all/event/pz_residual_rms": 17.876962661743164, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006975370459258556, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03278353810310364, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021052343770861626, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004989336244761944, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05968175083398819, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012892927043139935, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10281596332788467, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0018495229305699468, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.73988276720047, "metrics/particle/momentum/class_1/accepted/samples": 16288.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.043535489588975906, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12691497802734375, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11306364089250565, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08438905328512192, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02180999144911766, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.254915714263916, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004622770007699728, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23193921148777008, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.020840970799326897, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1960301399230957, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9653292298316956, "metrics/particle/momentum/class_1/event/energy_bias": 2.8696486949920654, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.694424629211426, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.668052673339844, "metrics/particle/momentum/class_1/event/px_bias": -0.3964247703552246, "metrics/particle/momentum/class_1/event/py_bias": -0.42071616649627686, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.3481364250183105, "metrics/particle/momentum/class_1/event/pz_bias": 0.6147053241729736, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.743111610412598, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03480980545282364, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1946020871400833, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19632020592689514, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026316717267036438, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23671582341194153, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02803030237555504, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5888781547546387, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.05329624563455582, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6020498275756836, "metrics/particle/momentum/class_2/accepted/samples": 1320.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8528407216072083, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6021177172660828, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 97129.109375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 97128.6328125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34507307410240173, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5017884.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07381041347980499, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9265899658203125, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.03920811787247658, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.740329742431641, "metrics/particle/momentum/class_2/assigned/samples": 2669.0, "metrics/particle/momentum/class_2/efficiency": 0.4480651617050171, "metrics/particle/momentum/class_2/event/energy_bias": -7.334629058837891, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.575450897216797, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.329019546508789, "metrics/particle/momentum/class_2/event/px_bias": 0.32336825132369995, "metrics/particle/momentum/class_2/event/py_bias": -0.37501153349876404, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.338927268981934, "metrics/particle/momentum/class_2/event/pz_bias": -0.12687352299690247, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.11058807373047, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019404271617531776, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1347760707139969, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13500161468982697, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.033987823873758316, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17620114982128143, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001023978111334145, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21339744329452515, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007892909459769726, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9850130677223206, "metrics/particle/momentum/class_3/accepted/samples": 11719.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30518195033073425, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3987658023834229, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.628173589706421, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4979615211486816, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.159093976020813, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.49867057800293, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0073698447085917, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5321063995361328, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018030088394880295, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.129737138748169, "metrics/particle/momentum/class_3/assigned/samples": 14790.0, "metrics/particle/momentum/class_3/efficiency": 0.7603815197944641, "metrics/particle/momentum/class_3/event/energy_bias": 4.915253639221191, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.942066192626953, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.605278491973877, "metrics/particle/momentum/class_3/event/px_bias": 0.37048786878585815, "metrics/particle/momentum/class_3/event/py_bias": 0.5687088370323181, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.630022048950195, "metrics/particle/momentum/class_3/event/pz_bias": 0.688739538192749, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.581748962402344, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011089091189205647, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.1061309427022934, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.053523752838373184, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0036496876273304224, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09852460771799088, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23917417228221893, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.01038892287760973, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8230646252632141, "metrics/particle/momentum/class_4/accepted/samples": 332.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.13343985378742218, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.205610752105713, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.7372850179672241, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6791247129440308, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.93445587158203, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3889787793159485, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.06715570390224457, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1449177265167236, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9171270728111267, "metrics/particle/momentum/class_4/event/energy_bias": -0.4638291895389557, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5394827127456665, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9778895974159241, "metrics/particle/momentum/class_4/event/px_bias": 0.10328797996044159, "metrics/particle/momentum/class_4/event/py_bias": -0.03673817962408066, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.005862832069397, "metrics/particle/momentum/class_4/event/pz_bias": 0.17556320130825043, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.947110176086426, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00499113230034709, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.021670928224921227, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017417239025235176, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008264250122010708, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0491134375333786, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18288885056972504, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05550573766231537, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7964416742324829, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04148777574300766, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03971857205033302, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.035334378480911255, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.005759649910032749, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13496503233909607, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5595279932022095, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2792293131351471, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.279426097869873, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.23064292967319489, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8741673231124878, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6181986927986145, "metrics/particle/momentum/class_5/event/px_bias": -0.011436156928539276, "metrics/particle/momentum/class_5/event/py_bias": 0.07088321447372437, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5076357126235962, "metrics/particle/momentum/class_5/event/pz_bias": -0.21619823575019836, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2571122646331787, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4870519638061523, "step_valid_time": 7949.087710857391, "step_total_time": 7950.574763536453, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025188098021938, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05567962357687073, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2901, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9621890547263682, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586913417052214, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9604370137394471, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04130865829477859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9563780568407139, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9581195166363184, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9621890547263682, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031646620552104, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0712338640096335, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9518716577540107, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9499666444296198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0481283422459893, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9471815357301376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9509803921568627, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9490771625528129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948069241011984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064931679185571, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07501042648989731, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1194, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1192, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9291828793774319, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9313572542901716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.930268796260226, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0686427457098284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9297971918876755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9287105570705103, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9291828793774319} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_6000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..46a59d8d11d058f77b835f059d9f0c0229c5f8e2 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28425241614553726, "train": {"Classification_binary": 3.349606990814209, "Classification": 0.029196787625551224, "Regression_pt": 0.18214376270771027, "Regression_eta": 0.009919758886098862, "Regression_sin_phi": 0.009777610190212727, "Regression_cos_phi": 0.011871226131916046, "Regression_energy": 0.1832249015569687, "Total": 11.306829452514648, "Total_unweighted": 3.775741037912667}, "train_group": {}, "valid": {"Classification_binary": 3.383080800374349, "Classification": 0.02850085993607839, "Regression_pt": 0.17974899212519327, "Regression_eta": 0.013193806012471518, "Regression_sin_phi": 0.010335213194290796, "Regression_cos_phi": 0.0103235791126887, "Regression_energy": 0.18209779262542725, "Total": 11.363652547200521, "Total_unweighted": 3.807281043380499, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.029699455946683884, "diagnostic/loss/clic_pf/Classification_binary": 3.372805595397949, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.010248720645904541, "diagnostic/loss/clic_pf/Regression_energy": 0.18600556254386902, "diagnostic/loss/clic_pf/Regression_eta": 0.012262493371963501, "diagnostic/loss/clic_pf/Regression_pt": 0.1839168816804886, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.010241309180855751, "diagnostic/regression/clic_pf/energy_pred_mean": 0.06764193624258041, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.22726812958717346, "diagnostic/regression/clic_pf/energy_residual_mean": 0.08319555968046188, "diagnostic/regression/clic_pf/energy_residual_rms": 0.505086362361908, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.02044915035367012, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2233908325433731, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0892936959862709, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5110225677490234, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9548571705818176, "metrics/particle/class_1/pid_accuracy": 0.9871554374694824, "metrics/particle/class_2/efficiency": 0.29984211921691895, "metrics/particle/class_2/pid_accuracy": 0.5908235907554626, "metrics/particle/class_3/efficiency": 0.6808492541313171, "metrics/particle/class_3/pid_accuracy": 0.979675829410553, "metrics/particle/class_4/efficiency": 0.9360465407371521, "metrics/particle/class_4/pid_accuracy": 0.17287784814834595, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.41803279519081116, "metrics/particle/count/bias_mean": -0.9143790602684021, "metrics/particle/count/mae": 3.98888897895813, "metrics/particle/count/prediction_mean": 69.08496856689453, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06798847764730453, "metrics/particle/event/energy_response_mean": 0.9982898235321045, "metrics/particle/event/met_abs_error": 8.203487396240234, "metrics/particle/event/scalar_pt_relative_abs_error": 0.055855169892311096, "metrics/particle/event/scalar_pt_response_mean": 1.0005525350570679, "metrics/particle/event/vector_pt_closure": 0.06531448662281036, "metrics/particle/matched/delta_eta_abs_mean": 0.017528779804706573, "metrics/particle/matched/delta_phi_abs_mean": 0.013138221576809883, "metrics/particle/matched/delta_r_mean": 0.024428579956293106, "metrics/particle/matched/energy_relative_abs_mean": 0.08925898373126984, "metrics/particle/matched/pid_accuracy": 0.9590384364128113, "metrics/particle/matched/pt_relative_abs_mean": 0.07933596521615982, "metrics/particle/matching/duplicate_fraction": 0.034701988101005554, "metrics/particle/matching/efficiency": 0.7818653583526611, "metrics/particle/matching/f1": 0.787005603313446, "metrics/particle/matching/purity": 0.7922137975692749, "metrics/particle/matching/target_coverage_dr0p05": 0.724105715751648, "metrics/particle/matching/target_coverage_dr0p10": 0.8189619183540344, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6914536952972412, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8285884857177734, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.027297094464302063, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08979889005422592, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07998836785554886, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.022155877202749252, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13671620190143585, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004014552570879459, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3926527202129364, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02881879173219204, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.331006407737732, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7971.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.36451414227485657, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2695342302322388, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3191429376602173, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1870447397232056, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1672787070274353, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 14.534818649291992, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.020003866404294968, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.044750452041626, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.12373219430446625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.433707237243652, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10348.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7415573596954346, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.175691843032837, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.848437309265137, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.479344367980957, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.844573974609375, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.5003163814544678, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.374635696411133, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.415522813796997, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.720324516296387, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03286770358681679, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10090319812297821, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09547960758209229, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02029377408325672, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1565474569797516, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01525746937841177, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8923097848892212, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.1317673921585083, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.689012050628662, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1573.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.859615683555603, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.11798095703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 47709.859375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 47709.68359375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.31418439745903015, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2533334.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08865248411893845, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.4780848026275635, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03341878205537796, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.323637962341309, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2820.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5120442509651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.9801389575004578, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.19748306274414, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.532499074935913, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.21577072143554688, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08841897547245026, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.977483749389648, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.056307584047317505, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.484764099121094, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022882649675011635, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0884179100394249, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07800016552209854, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014518321491777897, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1344531625509262, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007058312767185271, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.14266304671764374, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.04040459170937538, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.645319402217865, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18418.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20639516413211823, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8550278544425964, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5713971257209778, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.44914501905441284, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10685738176107407, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.48114538192749, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003973262384533882, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3460422158241272, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01460718922317028, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8436331748962402, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21393.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8389359712600708, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -2.533393144607544, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 10.739448547363281, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 8.9434814453125, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 2.2067503929138184, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.7591307759284973, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.070207595825195, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.953681230545044, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.34617805480957, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.0247027650475502, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08951394259929657, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07955023646354675, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017020422965288162, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1364341378211975, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024676346220076084, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2560977637767792, "metrics/particle/momentum/all/accepted/pz_bias": -0.04224148765206337, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0890737771987915, "metrics/particle/momentum/all/accepted/samples": 27962.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3070374131202698, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0821869373321533, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3893.628662109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3893.4990234375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1418651044368744, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 723642.1875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015682416036725044, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8108028769493103, "metrics/particle/momentum/all/assigned/pz_bias": 0.030732043087482452, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.295603275299072, "metrics/particle/momentum/all/assigned/samples": 34561.0, "metrics/particle/momentum/all/efficiency": 0.7816072702407837, "metrics/particle/momentum/all/event/energy_bias": -2.3378400802612305, "metrics/particle/momentum/all/event/p3_error_mean": 20.623815536499023, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.090540885925293, "metrics/particle/momentum/all/event/px_bias": 3.2670953273773193, "metrics/particle/momentum/all/event/py_bias": 2.3478660583496094, "metrics/particle/momentum/all/event/pz_abs_mean": 14.666851997375488, "metrics/particle/momentum/all/event/pz_bias": 2.425511598587036, "metrics/particle/momentum/all/event/pz_residual_rms": 24.19916343688965, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017837515100836754, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04213138669729233, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.026441359892487526, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008783917874097824, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06400278210639954, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010562943061813712, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15705808997154236, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.05938583239912987, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7550918459892273, "metrics/particle/momentum/class_1/accepted/samples": 16094.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07036799192428589, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1457194685935974, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12302606552839279, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08690451830625534, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.028270017355680466, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7720048427581787, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005393231753259897, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.33540278673171997, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0387507826089859, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.7024810314178467, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9538315534591675, "metrics/particle/momentum/class_1/event/energy_bias": 7.119150161743164, "metrics/particle/momentum/class_1/event/p3_error_mean": 12.64439582824707, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.383724212646484, "metrics/particle/momentum/class_1/event/px_bias": -0.06669992208480835, "metrics/particle/momentum/class_1/event/py_bias": -0.9807748794555664, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.571296691894531, "metrics/particle/momentum/class_1/event/pz_bias": -1.229964017868042, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.535402297973633, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04569310322403908, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.22181116044521332, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21801944077014923, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06657490879297256, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2568853497505188, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043429844081401825, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.9612046480178833, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.010524717159569263, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.0045599937438965, "metrics/particle/momentum/class_2/accepted/samples": 898.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.162847638130188, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9091495871543884, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 53641.2578125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 53640.65625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.469696968793869, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2686292.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09968101978302002, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.086732387542725, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.20497293770313263, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 10.418872833251953, "metrics/particle/momentum/class_2/assigned/samples": 2508.0, "metrics/particle/momentum/class_2/efficiency": 0.30482009053230286, "metrics/particle/momentum/class_2/event/energy_bias": -13.045075416564941, "metrics/particle/momentum/class_2/event/p3_error_mean": 19.313236236572266, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.988025665283203, "metrics/particle/momentum/class_2/event/px_bias": 2.2744946479797363, "metrics/particle/momentum/class_2/event/py_bias": 1.3960691690444946, "metrics/particle/momentum/class_2/event/pz_abs_mean": 14.104146003723145, "metrics/particle/momentum/class_2/event/pz_bias": 2.0105865001678467, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.749774932861328, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.033776406198740005, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1518726348876953, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1513097584247589, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03769582882523537, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1935325413942337, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001243186416104436, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2576827108860016, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0157347209751606, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9302619695663452, "metrics/particle/momentum/class_3/accepted/samples": 10457.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4414807856082916, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.2053356170654297, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.2606756687164307, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.102205514907837, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2204154133796692, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 26.811872482299805, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01366493571549654, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.8086768388748169, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.08928541839122772, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.042804718017578, "metrics/particle/momentum/class_3/assigned/samples": 14636.0, "metrics/particle/momentum/class_3/efficiency": 0.6784972548484802, "metrics/particle/momentum/class_3/event/energy_bias": 6.430723190307617, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.996222496032715, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.641470432281494, "metrics/particle/momentum/class_3/event/px_bias": 0.9820126295089722, "metrics/particle/momentum/class_3/event/py_bias": 1.7464731931686401, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.845734596252441, "metrics/particle/momentum/class_3/event/pz_bias": 1.7280491590499878, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.112451553344727, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.0202596765011549, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.09341341257095337, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.046803247183561325, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0034621201921254396, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.0895800068974495, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.31169387698173523, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.11053398996591568, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9776709675788879, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11963828653097153, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0369738340377808, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7022452354431152, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6399929523468018, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.3116679191589355, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4276193082332611, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.07610934227705002, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2816956043243408, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -1.4481412172317505, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3757877349853516, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5589029788970947, "metrics/particle/momentum/class_4/event/px_bias": -0.030403895303606987, "metrics/particle/momentum/class_4/event/py_bias": 0.2142694592475891, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.5116666555404663, "metrics/particle/momentum/class_4/event/pz_bias": 0.2292238175868988, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.570997714996338, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01481432281434536, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03514524921774864, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028717778623104095, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007451377809047699, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07306608557701111, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.4114905893802643, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1874273717403412, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.1090456247329712, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01624981127679348, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0544397234916687, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0470108836889267, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010884354822337627, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1406371146440506, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6746710538864136, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.42488884925842285, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6587939262390137, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -1.3944966793060303, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8912193775177002, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.3084001541137695, "metrics/particle/momentum/class_5/event/px_bias": 0.10769180953502655, "metrics/particle/momentum/class_5/event/py_bias": -0.02817092277109623, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.090031385421753, "metrics/particle/momentum/class_5/event/pz_bias": -0.31238383054733276, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.648622989654541, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.1272902488708496, "step_valid_time": 1271.019377231598, "step_total_time": 1272.1466677188873, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017962758706953, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11058870862798786, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3008, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2816, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2799, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9339966832504146, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9361702127659575, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9350821849576623, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06382978723404255, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9283582089552239, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9305186170212766, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9294371575626764, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9339966832504146, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962373478010383, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.12216062699813124, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2060, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9143364403018198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9221128021486124, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9182081568977045, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07788719785138765, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9121171770972037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9198746642793196, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.91597949632271, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9143364403018198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954922755714808, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1265463586695924, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1269, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1126, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1117, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8762645914396887, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8873128447596532, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8817541111981206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11268715524034673, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8692607003891051, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.880220646178093, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.874706342991386, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8762645914396887} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_7000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..029aff6daf65cade045bfeb5fd2f7ee7dd9f4af7 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.3316278188364601, "train": {"Classification_binary": 3.27700138092041, "Classification": 0.027112560346722603, "Regression_pt": 0.17355424165725708, "Regression_eta": 0.012379760853946209, "Regression_sin_phi": 0.011212395504117012, "Regression_cos_phi": 0.010914753191173077, "Regression_energy": 0.17454740405082703, "Total": 11.085662841796875, "Total_unweighted": 3.686722496524453}, "train_group": {}, "valid": {"Classification_binary": 3.2556467056274414, "Classification": 0.027285493910312653, "Regression_pt": 0.16947891314824423, "Regression_eta": 0.012613637993733088, "Regression_sin_phi": 0.010033919165531794, "Regression_cos_phi": 0.01020274187127749, "Regression_energy": 0.17249945799509683, "Total": 10.860799153645834, "Total_unweighted": 3.6577608697116375, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.028044018894433975, "diagnostic/loss/clic_pf/Classification_binary": 3.2606868743896484, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00994933769106865, "diagnostic/loss/clic_pf/Regression_energy": 0.175194650888443, "diagnostic/loss/clic_pf/Regression_eta": 0.011835881508886814, "diagnostic/loss/clic_pf/Regression_pt": 0.172196626663208, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009868575260043144, "diagnostic/regression/clic_pf/energy_pred_mean": 0.059188298881053925, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.22055259346961975, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0747419148683548, "diagnostic/regression/clic_pf/energy_residual_rms": 0.49106690287590027, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.017711199820041656, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.21794456243515015, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08655572682619095, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4964155852794647, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9540620446205139, "metrics/particle/class_1/pid_accuracy": 0.9852904677391052, "metrics/particle/class_2/efficiency": 0.34179070591926575, "metrics/particle/class_2/pid_accuracy": 0.6139887571334839, "metrics/particle/class_3/efficiency": 0.6928333044052124, "metrics/particle/class_3/pid_accuracy": 0.9791874289512634, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.22523461282253265, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.4836065471172333, "metrics/particle/count/bias_mean": -0.34705883264541626, "metrics/particle/count/mae": 3.863398790359497, "metrics/particle/count/prediction_mean": 69.65229034423828, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06719227880239487, "metrics/particle/event/energy_response_mean": 1.0088344812393188, "metrics/particle/event/met_abs_error": 7.114103317260742, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05283127725124359, "metrics/particle/event/scalar_pt_response_mean": 1.0092612504959106, "metrics/particle/event/vector_pt_closure": 0.057969141751527786, "metrics/particle/matched/delta_eta_abs_mean": 0.013032312504947186, "metrics/particle/matched/delta_phi_abs_mean": 0.015780674293637276, "metrics/particle/matched/delta_r_mean": 0.022815151140093803, "metrics/particle/matched/energy_relative_abs_mean": 0.08777176588773727, "metrics/particle/matched/pid_accuracy": 0.9581649303436279, "metrics/particle/matched/pt_relative_abs_mean": 0.08017168939113617, "metrics/particle/matching/duplicate_fraction": 0.034963592886924744, "metrics/particle/matching/efficiency": 0.7900913953781128, "metrics/particle/matching/f1": 0.7920549511909485, "metrics/particle/matching/purity": 0.7940282225608826, "metrics/particle/matching/target_coverage_dr0p05": 0.7268508672714233, "metrics/particle/matching/target_coverage_dr0p10": 0.8268517851829529, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6955060362815857, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8356193900108337, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024335060268640518, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08468582481145859, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07978826016187668, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01760510355234146, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13606901466846466, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003692762227728963, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.38099631667137146, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.02769722230732441, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3502951860427856, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8124.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.36094895005226135, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1956124305725098, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3038454055786133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1725114583969116, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15779340267181396, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.713064193725586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.018292095512151718, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9385996460914612, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.016080515459179878, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.5156147480010986, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10387.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7557912468910217, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.3998320698738098, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.126214981079102, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.1583356857299805, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3475370705127716, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5773530006408691, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.758254051208496, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.7652815580368042, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.359075546264648, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03275827690958977, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09756958484649658, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09760291129350662, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01920553669333458, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1584104597568512, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019065190106630325, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0098536014556885, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16287867724895477, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6685969829559326, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1626.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8532673716545105, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.181057929992676, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 139105.171875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 139105.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3022598922252655, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 7402461.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09074858576059341, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.533226490020752, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.1793595254421234, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.195869445800781, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2832.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.529296875, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.38609638810157776, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.903339385986328, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4092463254928589, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.18081088364124298, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.14048384130001068, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.720182418823242, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.6005699634552, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.53687286376953, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021144932135939598, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08767206966876984, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07832169532775879, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014992664568126202, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1357368677854538, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008649115916341543, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12426432967185974, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.015184787102043629, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6183330416679382, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18499.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.19903284311294556, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.903644859790802, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.6828959584236145, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.565161406993866, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10752638429403305, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 13.557219505310059, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0035330760292708874, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3300640881061554, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.016197051852941513, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.4127938747406006, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21511.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8426254987716675, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.0026512322947382927, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.246337890625, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.582499027252197, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.29277753829956055, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9692206382751465, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.589942932128906, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.32926470041275024, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.216760635375977, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02273082546889782, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0873829647898674, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07985328137874603, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015986455604434013, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1372382938861847, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002725760219618678, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24907083809375763, "metrics/particle/momentum/all/accepted/pz_bias": -0.027284376323223114, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2447487115859985, "metrics/particle/momentum/all/accepted/samples": 28249.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.3008069396018982, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0951306819915771, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 11343.912109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 11343.787109375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1384393870830536, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 2113831.75, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015059026889503002, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7732606530189514, "metrics/particle/momentum/all/assigned/pz_bias": -0.029467010870575905, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.186106204986572, "metrics/particle/momentum/all/assigned/samples": 34730.0, "metrics/particle/momentum/all/efficiency": 0.7896296381950378, "metrics/particle/momentum/all/event/energy_bias": 0.016386955976486206, "metrics/particle/momentum/all/event/p3_error_mean": 19.882694244384766, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.863497734069824, "metrics/particle/momentum/all/event/px_bias": 0.8211255073547363, "metrics/particle/momentum/all/event/py_bias": -1.6870574951171875, "metrics/particle/momentum/all/event/pz_abs_mean": 15.070269584655762, "metrics/particle/momentum/all/event/pz_bias": -3.0365867614746094, "metrics/particle/momentum/all/event/pz_residual_rms": 25.961849212646484, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013480293564498425, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.038288939744234085, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02617556042969227, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008710555732250214, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0666617900133133, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0009943447075784206, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11710783839225769, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0016214543720707297, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.6560348868370056, "metrics/particle/momentum/class_1/accepted/samples": 16091.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06387769430875778, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.30111271142959595, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.28977471590042114, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.258464515209198, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.029751675203442574, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 14.014699935913086, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005037634167820215, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29767224192619324, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02756172977387905, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1227102279663086, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9536537528038025, "metrics/particle/momentum/class_1/event/energy_bias": 8.81154727935791, "metrics/particle/momentum/class_1/event/p3_error_mean": 13.0050687789917, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.039194107055664, "metrics/particle/momentum/class_1/event/px_bias": 0.000608399510383606, "metrics/particle/momentum/class_1/event/py_bias": -0.5663399696350098, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.359813690185547, "metrics/particle/momentum/class_1/event/pz_bias": 0.12196207046508789, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.40224838256836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04445100948214531, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.2159460484981537, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2145831137895584, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07436197996139526, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.25261616706848145, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.043043043464422226, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.0673985481262207, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.17493049800395966, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.6030144691467285, "metrics/particle/momentum/class_2/accepted/samples": 999.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1430937051773071, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9087929129600525, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 152629.1875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 152628.578125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.44672608375549316, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 7754053.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09996125847101212, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.958775520324707, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2608208656311035, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.978666305541992, "metrics/particle/momentum/class_2/assigned/samples": 2581.0, "metrics/particle/momentum/class_2/efficiency": 0.3391038775444031, "metrics/particle/momentum/class_2/event/energy_bias": -12.895355224609375, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.507930755615234, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.130245208740234, "metrics/particle/momentum/class_2/event/px_bias": 0.028817884624004364, "metrics/particle/momentum/class_2/event/py_bias": -1.0419412851333618, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.990875244140625, "metrics/particle/momentum/class_2/event/pz_bias": -1.945526123046875, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.279457092285156, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.035062383860349655, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1500369757413864, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1501799076795578, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03597995638847351, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19223710894584656, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001690934761427343, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.27672526240348816, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.053844153881073, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.1946873664855957, "metrics/particle/momentum/class_3/accepted/samples": 10645.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.43252143263816833, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0326895713806152, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.227637529373169, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0749034881591797, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21253053843975067, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.427349090576172, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012082541361451149, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7708576321601868, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.05096304789185524, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.9693126678466797, "metrics/particle/momentum/class_3/assigned/samples": 14732.0, "metrics/particle/momentum/class_3/efficiency": 0.690695583820343, "metrics/particle/momentum/class_3/event/energy_bias": 5.8266425132751465, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.033214569091797, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.487520217895508, "metrics/particle/momentum/class_3/event/px_bias": 0.5914438962936401, "metrics/particle/momentum/class_3/event/py_bias": -0.2142597883939743, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.991514205932617, "metrics/particle/momentum/class_3/event/pz_bias": -1.5429686307907104, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.266769409179688, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01627064310014248, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.10114474594593048, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04947128891944885, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0018412744393572211, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.08708971738815308, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2811274528503418, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.030927749350667, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9736468195915222, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10887053608894348, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8076469898223877, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.2338241338729858, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1784453392028809, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.48851203918457, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4348480999469757, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.009189236909151077, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.4363211393356323, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.9131532311439514, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.34737491607666, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5911314487457275, "metrics/particle/momentum/class_4/event/px_bias": 0.07173392176628113, "metrics/particle/momentum/class_4/event/py_bias": 0.1066633015871048, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4384976625442505, "metrics/particle/momentum/class_4/event/pz_bias": 0.4515947103500366, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.566949844360352, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011842734180390835, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03176647424697876, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.027929725125432014, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.011698895134031773, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06971778720617294, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.25917547941207886, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.039271168410778046, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9636551737785339, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.041620925068855286, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04214147850871086, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03830237314105034, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0008907685987651348, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12323149293661118, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5573006868362427, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.33596500754356384, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.146739959716797, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.813294529914856, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7445111274719238, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2986215353012085, "metrics/particle/momentum/class_5/event/px_bias": 0.1285213977098465, "metrics/particle/momentum/class_5/event/py_bias": 0.028820112347602844, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9333491921424866, "metrics/particle/momentum/class_5/event/pz_bias": -0.12164875119924545, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.88498854637146, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.294252872467041, "step_valid_time": 1464.8422157764435, "step_total_time": 1466.136468887329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025120631870799, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0946187626653332, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3000, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2830, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2816, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9386401326699834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9433333333333334, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.940980881130507, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056666666666666664, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9339966832504146, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9386666666666666, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9363258520365753, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9386401326699834, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012388639979601, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10996748995636352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2060, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2054, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9143364403018198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9200535953550693, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9171861086375779, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07994640464493077, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9116733244562805, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9173738276016079, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9145146927871772, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9143364403018198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974531919871938, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11713542413432798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1145, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1131, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8910505836575876, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8973354231974922, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8941819601718078, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10266457680250783, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8801556420233463, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8863636363636364, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.883248730964467, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8910505836575876} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_8000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..0029e39cb26aa4180988bbb1ea96ef6b81a41fc7 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.37900322152738297, "train": {"Classification_binary": 3.167433977127075, "Classification": 0.02876991406083107, "Regression_pt": 0.17065460979938507, "Regression_eta": 0.010351905599236488, "Regression_sin_phi": 0.00924263708293438, "Regression_cos_phi": 0.00901494175195694, "Regression_energy": 0.17193442583084106, "Total": 10.610542297363281, "Total_unweighted": 3.56740241125226}, "train_group": {}, "valid": {"Classification_binary": 3.1635335286458335, "Classification": 0.02670192966858546, "Regression_pt": 0.16840424140294394, "Regression_eta": 0.012805722653865814, "Regression_sin_phi": 0.01002089430888494, "Regression_cos_phi": 0.00989354650179545, "Regression_energy": 0.17102875312169394, "Total": 10.703557332356771, "Total_unweighted": 3.562388616303603, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.027641506865620613, "diagnostic/loss/clic_pf/Classification_binary": 3.1540379524230957, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009636368602514267, "diagnostic/loss/clic_pf/Regression_energy": 0.1735590100288391, "diagnostic/loss/clic_pf/Regression_eta": 0.012119026854634285, "diagnostic/loss/clic_pf/Regression_pt": 0.17052821815013885, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009830116294324398, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04028215631842613, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2133464366197586, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0558357760310173, "diagnostic/regression/clic_pf/energy_residual_rms": 0.48595550656318665, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.001398436725139618, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2121569663286209, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0702429711818695, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4924567937850952, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9571431279182434, "metrics/particle/class_1/pid_accuracy": 0.9868746399879456, "metrics/particle/class_2/efficiency": 0.3419034779071808, "metrics/particle/class_2/pid_accuracy": 0.6484168767929077, "metrics/particle/class_3/efficiency": 0.7156374454498291, "metrics/particle/class_3/pid_accuracy": 0.9809650182723999, "metrics/particle/class_4/efficiency": 0.9389534592628479, "metrics/particle/class_4/pid_accuracy": 0.20227038860321045, "metrics/particle/class_5/efficiency": 0.9798792600631714, "metrics/particle/class_5/pid_accuracy": 0.48459959030151367, "metrics/particle/count/bias_mean": 0.1366013139486313, "metrics/particle/count/mae": 3.8006536960601807, "metrics/particle/count/prediction_mean": 70.13594818115234, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0649498701095581, "metrics/particle/event/energy_response_mean": 0.988277792930603, "metrics/particle/event/met_abs_error": 6.777454376220703, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05211533233523369, "metrics/particle/event/scalar_pt_response_mean": 1.002638339996338, "metrics/particle/event/vector_pt_closure": 0.05612189695239067, "metrics/particle/matched/delta_eta_abs_mean": 0.013917039148509502, "metrics/particle/matched/delta_phi_abs_mean": 0.013587137684226036, "metrics/particle/matched/delta_r_mean": 0.021726908162236214, "metrics/particle/matched/energy_relative_abs_mean": 0.08569630980491638, "metrics/particle/matched/pid_accuracy": 0.9609277844429016, "metrics/particle/matched/pt_relative_abs_mean": 0.07934532314538956, "metrics/particle/matching/duplicate_fraction": 0.0343683585524559, "metrics/particle/matching/efficiency": 0.801510751247406, "metrics/particle/matching/f1": 0.8007294535636902, "metrics/particle/matching/purity": 0.7999497056007385, "metrics/particle/matching/target_coverage_dr0p05": 0.7371964454650879, "metrics/particle/matching/target_coverage_dr0p10": 0.8347417116165161, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7093810439109802, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8473188281059265, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02393173985183239, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.0836644172668457, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07883942127227783, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0085090147331357, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13538619875907898, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003072007792070508, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.35893315076828003, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.045776888728141785, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2659307718276978, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8138.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3596388101577759, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0348625183105469, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1964048147201538, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0504851341247559, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15326076745986938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.123372077941895, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01762835867702961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9331003427505493, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.12609335780143738, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.865828275680542, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10381.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7570936679840088, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.176159381866455, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.099080085754395, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.008106231689453, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.06216657906770706, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7319888472557068, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.844985485076904, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.0919699668884277, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.994488716125488, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02899700403213501, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09523164480924606, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09391351044178009, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015096788294613361, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15093401074409485, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.014058679342269897, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8783814907073975, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.16682997345924377, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.7505764961242676, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1636.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8435064554214478, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5587118864059448, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 11713.779296875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 11713.5849609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.29399293661117554, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 622968.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07879858464002609, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.379279375076294, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05611289665102959, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.056962966918945, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2830.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5325520634651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.642104148864746, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.230506896972656, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4863572120666504, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.07959388196468353, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05924150347709656, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.046075820922852, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -3.24458384513855, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.467880249023438, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.020249828696250916, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08626596629619598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07866093516349792, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007813047617673874, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13469994068145752, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006350550102069974, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1303778737783432, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.042626745998859406, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6201292872428894, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18896.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17253226041793823, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8896742463111877, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7047044634819031, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5828651189804077, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.095782071352005, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 12.471050262451172, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028674497734755278, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.31243273615837097, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05909641459584236, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.2368624210357666, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21622.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8607087731361389, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.0504097938537598, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.29589557647705, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.559940338134766, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.41989558935165405, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6991475820541382, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.642086029052734, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.9444741010665894, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.469404220581055, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02179408259689808, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08603912591934204, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07958196103572845, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.00842623133212328, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13587231934070587, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0020927798468619585, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23793688416481018, "metrics/particle/momentum/all/accepted/pz_bias": -0.05060834065079689, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0677419900894165, "metrics/particle/momentum/all/accepted/samples": 28670.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.28280729055404663, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9872994422912598, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 952.4776000976562, "metrics/particle/momentum/all/assigned/pt_relative_bias": 952.3427124023438, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12901559472084045, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 177567.734375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01343553513288498, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7465707063674927, "metrics/particle/momentum/all/assigned/pz_bias": -0.07882058620452881, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.18239164352417, "metrics/particle/momentum/all/assigned/samples": 34833.0, "metrics/particle/momentum/all/efficiency": 0.8013976216316223, "metrics/particle/momentum/all/event/energy_bias": -4.767853260040283, "metrics/particle/momentum/all/event/p3_error_mean": 19.629703521728516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.469291687011719, "metrics/particle/momentum/all/event/px_bias": -0.4024682641029358, "metrics/particle/momentum/all/event/py_bias": 0.09208275377750397, "metrics/particle/momentum/all/event/pz_abs_mean": 14.992040634155273, "metrics/particle/momentum/all/event/pz_bias": -6.281027793884277, "metrics/particle/momentum/all/event/pz_residual_rms": 24.173236846923828, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011914105154573917, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03730160742998123, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.026624782010912895, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0014024910051375628, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0656798779964447, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011160714784637094, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13652129471302032, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.046064089983701706, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.766990602016449, "metrics/particle/momentum/class_1/accepted/samples": 16128.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.061978474259376526, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1295822560787201, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.13401032984256744, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09174167364835739, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.027618087828159332, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.307723045349121, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005215432960540056, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3083861470222473, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0483691580593586, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.715588092803955, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9558466076850891, "metrics/particle/momentum/class_1/event/energy_bias": 3.5350892543792725, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.734705924987793, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.223513603210449, "metrics/particle/momentum/class_1/event/px_bias": -1.1470060348510742, "metrics/particle/momentum/class_1/event/py_bias": -0.40948301553726196, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.651812553405762, "metrics/particle/momentum/class_1/event/pz_bias": -1.5806044340133667, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.219566345214844, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04538999870419502, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21215680241584778, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20789207518100739, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06036815047264099, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24703676998615265, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02777777798473835, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7331067323684692, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.13828185200691223, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.683532238006592, "metrics/particle/momentum/class_2/accepted/samples": 1008.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0987766981124878, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8553103804588318, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 12578.0478515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 12577.443359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4299810230731964, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 645608.4375, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0903225839138031, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.90403151512146, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.22765816748142242, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.486306190490723, "metrics/particle/momentum/class_2/assigned/samples": 2635.0, "metrics/particle/momentum/class_2/efficiency": 0.34215885400772095, "metrics/particle/momentum/class_2/event/energy_bias": -12.582674026489258, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.880779266357422, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.184738159179688, "metrics/particle/momentum/class_2/event/px_bias": 0.5707568526268005, "metrics/particle/momentum/class_2/event/py_bias": -0.2476062774658203, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.132562637329102, "metrics/particle/momentum/class_2/event/pz_bias": -3.8188424110412598, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.32233428955078, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03444284945726395, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14665597677230835, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14721128344535828, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.025626730173826218, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1891995370388031, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011802087537944317, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24549873173236847, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.045716818422079086, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9013349413871765, "metrics/particle/momentum/class_3/accepted/samples": 11015.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.39752885699272156, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.98212468624115, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.154024839401245, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9940134286880493, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19443880021572113, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.87372589111328, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009403964504599571, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6925739645957947, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.07965978235006332, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.066376209259033, "metrics/particle/momentum/class_3/assigned/samples": 14781.0, "metrics/particle/momentum/class_3/efficiency": 0.714702844619751, "metrics/particle/momentum/class_3/event/energy_bias": 6.668487071990967, "metrics/particle/momentum/class_3/event/p3_error_mean": 12.430910110473633, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.446124076843262, "metrics/particle/momentum/class_3/event/px_bias": -0.01408550888299942, "metrics/particle/momentum/class_3/event/py_bias": 0.5582483410835266, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.247187614440918, "metrics/particle/momentum/class_3/event/pz_bias": -1.0188809633255005, "metrics/particle/momentum/class_3/event/pz_residual_rms": 14.797494888305664, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015950119122862816, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0905357301235199, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04910390451550484, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.004422441590577364, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09085817635059357, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.29103776812553406, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.034691743552684784, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9677113890647888, "metrics/particle/momentum/class_4/accepted/samples": 340.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08674008399248123, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7817156314849854, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.076979160308838, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.02244234085083, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.599954605102539, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.43997302651405334, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.13812877237796783, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.5102956295013428, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.939226508140564, "metrics/particle/momentum/class_4/event/energy_bias": -1.238532543182373, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3278322219848633, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5371299982070923, "metrics/particle/momentum/class_4/event/px_bias": 0.09826700389385223, "metrics/particle/momentum/class_4/event/py_bias": 0.08016279339790344, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.460080623626709, "metrics/particle/momentum/class_4/event/pz_bias": 0.44402313232421875, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.551452159881592, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011852025054395199, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02843337319791317, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.0247347392141819, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.005380210466682911, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07807566225528717, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.005586592014878988, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3896198868751526, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.29757237434387207, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.1917102336883545, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.014906189404428005, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04203270375728607, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03839482367038727, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.011408466845750809, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13476233184337616, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6514725089073181, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5609423518180847, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.724611759185791, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -1.150222659111023, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7126014232635498, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.2527616024017334, "metrics/particle/momentum/class_5/event/px_bias": 0.08959939330816269, "metrics/particle/momentum/class_5/event/py_bias": 0.11076094955205917, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9423622488975525, "metrics/particle/momentum/class_5/event/pz_bias": -0.3067231774330139, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.882519245147705, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3686161041259766, "step_valid_time": 1654.6754064559937, "step_total_time": 1656.0440230369568, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990390614483231, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09301595155207387, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2847, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9442786069651742, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.941468253968254, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9428713363139593, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05853174603174603, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9399668325041459, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9371693121693122, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9385659877463156, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9442786069651742, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9916585882431607, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10182196509034802, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2071, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2063, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9192188193519751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9278673835125448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9235228539576366, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0721326164874552, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9156679982245894, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9242831541218638, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9199554069119287, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9192188193519751, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9936154602058156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10748093385163471, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1146, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1137, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8918287937743191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8967136150234741, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8942645337495123, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10328638497652583, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8848249027237354, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8896713615023474, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.887241513850956, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8918287937743191} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_9000.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..817e37d30a0defbfba5b37e56293e3cedf7fd7d7 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.42637862421830586, "train": {"Classification_binary": 3.055936813354492, "Classification": 0.026950985193252563, "Regression_pt": 0.14828263223171234, "Regression_eta": 0.010921071283519268, "Regression_sin_phi": 0.009103975258767605, "Regression_cos_phi": 0.00950602162629366, "Regression_energy": 0.14803853631019592, "Total": 9.82799243927002, "Total_unweighted": 3.4087400352582335}, "train_group": {}, "valid": {"Classification_binary": 3.132639249165853, "Classification": 0.02610931545495987, "Regression_pt": 0.1624621550242106, "Regression_eta": 0.012851736197868982, "Regression_sin_phi": 0.009815831979115805, "Regression_cos_phi": 0.00982277219494184, "Regression_energy": 0.16530258456865946, "Total": 10.470398585001627, "Total_unweighted": 3.5190036445856094, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.026679666712880135, "diagnostic/loss/clic_pf/Classification_binary": 3.1325666904449463, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00953687634319067, "diagnostic/loss/clic_pf/Regression_energy": 0.16724611818790436, "diagnostic/loss/clic_pf/Regression_eta": 0.011888369917869568, "diagnostic/loss/clic_pf/Regression_pt": 0.1643272042274475, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009674357250332832, "diagnostic/regression/clic_pf/energy_pred_mean": 0.06160799041390419, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.21501904726028442, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07716161012649536, "diagnostic/regression/clic_pf/energy_residual_rms": 0.47047194838523865, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.02267175167798996, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.21310517191886902, "diagnostic/regression/clic_pf/pt_residual_mean": 0.09151628613471985, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4779725968837738, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9575406908988953, "metrics/particle/class_1/pid_accuracy": 0.9861742258071899, "metrics/particle/class_2/efficiency": 0.35870546102523804, "metrics/particle/class_2/pid_accuracy": 0.6171015501022339, "metrics/particle/class_3/efficiency": 0.6995365619659424, "metrics/particle/class_3/pid_accuracy": 0.9808042049407959, "metrics/particle/class_4/efficiency": 0.9273256063461304, "metrics/particle/class_4/pid_accuracy": 0.22361546754837036, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.5306122303009033, "metrics/particle/count/bias_mean": 1.0496731996536255, "metrics/particle/count/mae": 3.6901960372924805, "metrics/particle/count/prediction_mean": 71.04901885986328, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06547233462333679, "metrics/particle/event/energy_response_mean": 1.0238310098648071, "metrics/particle/event/met_abs_error": 6.134605884552002, "metrics/particle/event/scalar_pt_relative_abs_error": 0.054848868399858475, "metrics/particle/event/scalar_pt_response_mean": 1.0305625200271606, "metrics/particle/event/vector_pt_closure": 0.0560719296336174, "metrics/particle/matched/delta_eta_abs_mean": 0.01642356626689434, "metrics/particle/matched/delta_phi_abs_mean": 0.01333688572049141, "metrics/particle/matched/delta_r_mean": 0.023655924946069717, "metrics/particle/matched/energy_relative_abs_mean": 0.09045139700174332, "metrics/particle/matched/pid_accuracy": 0.9591807723045349, "metrics/particle/matched/pt_relative_abs_mean": 0.08250463753938675, "metrics/particle/matching/duplicate_fraction": 0.03653925657272339, "metrics/particle/matching/efficiency": 0.7960298657417297, "metrics/particle/matching/f1": 0.7901058197021484, "metrics/particle/matching/purity": 0.7842693328857422, "metrics/particle/matching/target_coverage_dr0p05": 0.7095771431922913, "metrics/particle/matching/target_coverage_dr0p10": 0.8354699611663818, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6956367492675781, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8455727696418762, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.025058168917894363, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08846447616815567, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08473348617553711, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.03476117551326752, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.14400063455104828, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004108054097741842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.37237226963043213, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.05553467199206352, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.3896803855895996, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8033.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3637583553791046, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.126619815826416, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3222911357879639, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.2000371217727661, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1645992398262024, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.753390312194824, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016984732821583748, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9166371822357178, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08463572710752487, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4399189949035645, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10480.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7473253607749939, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.37758389115333557, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.047735214233398, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.0166544914245605, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1392124593257904, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.07631774246692657, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.789432525634766, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.2423129081726074, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 15.273959159851074, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03195983171463013, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09452590346336365, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09625330567359924, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03033386915922165, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15678420662879944, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.015047022141516209, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.873128354549408, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.15874266624450684, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.128700017929077, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1595.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.906891405582428, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.0909838676452637, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 32784.3046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 32784.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3170563876628876, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1767698.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08356258273124695, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.637963056564331, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2605212926864624, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.533907890319824, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2908.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5192057490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 0.4092590808868408, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.374031066894531, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.4870219230651855, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.004275377374142408, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1685577929019928, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.179420471191406, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 4.268285274505615, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 22.223886489868164, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022284096106886864, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09128420054912567, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08025582134723663, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024832595139741898, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13837659358978271, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008520609117113054, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1315700262784958, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.04433742165565491, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6463736295700073, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18778.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1774277240037918, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7752362489700317, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5812722444534302, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47692498564720154, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10414746403694153, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.269289493560791, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003456221194937825, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3061807453632355, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.07409828901290894, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7335790395736694, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21700.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8553338646888733, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.415925025939941, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.68464469909668, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.615932464599609, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.6639606952667236, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.11789530515670776, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.151809215545654, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.0350728034973145, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.188911437988281, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023611875250935555, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09066881984472275, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08242033421993256, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.027949219569563866, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14107537269592285, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002569879638031125, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2413056492805481, "metrics/particle/momentum/all/accepted/pz_bias": 0.05392778292298317, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1713074445724487, "metrics/particle/momentum/all/accepted/samples": 28406.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.293536514043808, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9892321825027466, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2717.830322265625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2717.71435546875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.13984838128089905, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 508892.40625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.014135886915028095, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7646396160125732, "metrics/particle/momentum/all/assigned/pz_bias": 0.09269582480192184, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.291015148162842, "metrics/particle/momentum/all/assigned/samples": 35088.0, "metrics/particle/momentum/all/efficiency": 0.7940181493759155, "metrics/particle/momentum/all/event/energy_bias": 5.202767372131348, "metrics/particle/momentum/all/event/p3_error_mean": 19.54501724243164, "metrics/particle/momentum/all/event/pt_vector_error_mean": 9.378945350646973, "metrics/particle/momentum/all/event/px_bias": -2.807448387145996, "metrics/particle/momentum/all/event/py_bias": 0.12698021531105042, "metrics/particle/momentum/all/event/pz_abs_mean": 15.14199447631836, "metrics/particle/momentum/all/event/pz_bias": 7.545670509338379, "metrics/particle/momentum/all/event/pz_residual_rms": 24.777708053588867, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.012120588682591915, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04317527264356613, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.027609650045633316, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.011381935328245163, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07018370926380157, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0009922480676323175, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11851681768894196, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.020971501246094704, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7153041958808899, "metrics/particle/momentum/class_1/accepted/samples": 16125.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06252924352884293, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16354624927043915, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1732630878686905, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.14454242587089539, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.028686581179499626, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.0592122077941895, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004919392988085747, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2809258699417114, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02424631454050541, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3358471393585205, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9556688070297241, "metrics/particle/momentum/class_1/event/energy_bias": 6.476810932159424, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.793715476989746, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.095060348510742, "metrics/particle/momentum/class_1/event/px_bias": -0.5680387020111084, "metrics/particle/momentum/class_1/event/py_bias": -0.9865390658378601, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.712980270385742, "metrics/particle/momentum/class_1/event/pz_bias": 0.5232840776443481, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.844932556152344, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04896554723381996, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20916332304477692, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20675814151763916, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06952111423015594, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24702419340610504, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03846153989434242, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8267766237258911, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.29141107201576233, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9342892169952393, "metrics/particle/momentum/class_2/accepted/samples": 1040.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1192529201507568, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.9808850288391113, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 35008.40234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 35007.81640625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.43628349900245667, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1826759.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09658464789390564, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.024896621704102, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.5023406147956848, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 11.186525344848633, "metrics/particle/momentum/class_2/assigned/samples": 2723.0, "metrics/particle/momentum/class_2/efficiency": 0.3530210554599762, "metrics/particle/momentum/class_2/event/energy_bias": -10.893056869506836, "metrics/particle/momentum/class_2/event/p3_error_mean": 17.784090042114258, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.732810974121094, "metrics/particle/momentum/class_2/event/px_bias": -1.5353472232818604, "metrics/particle/momentum/class_2/event/py_bias": 1.295888900756836, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.490912437438965, "metrics/particle/momentum/class_2/event/pz_bias": 4.081116676330566, "metrics/particle/momentum/class_2/event/pz_residual_rms": 22.035327911376953, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.038880109786987305, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15081529319286346, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15442831814289093, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.06323250383138657, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1973835676908493, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001584638375788927, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2694162428379059, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.08069087564945221, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.151011347770691, "metrics/particle/momentum/class_3/accepted/samples": 10728.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.4112706482410431, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9257416725158691, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.145401954650879, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.0137901306152344, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.2145293951034546, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.755874633789062, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009900327771902084, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7282830476760864, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.10115968436002731, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.7513821125030518, "metrics/particle/momentum/class_3/assigned/samples": 14949.0, "metrics/particle/momentum/class_3/efficiency": 0.6960809826850891, "metrics/particle/momentum/class_3/event/energy_bias": 11.364895820617676, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.724555015563965, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.621267318725586, "metrics/particle/momentum/class_3/event/px_bias": -0.8421779870986938, "metrics/particle/momentum/class_3/event/py_bias": -0.27952247858047485, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.585433006286621, "metrics/particle/momentum/class_3/event/pz_bias": 2.519932508468628, "metrics/particle/momentum/class_3/event/pz_residual_rms": 18.2806339263916, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01573612540960312, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.11335374414920807, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.057513780891895294, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.010352615267038345, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10039090365171432, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.30531951785087585, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.029193244874477386, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9418085217475891, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1288474202156067, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3354443311691284, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.9480157494544983, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.8966529965400696, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.163483619689941, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.399846613407135, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.016549836844205856, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3042911291122437, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -1.0342111587524414, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.3023574352264404, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5038574934005737, "metrics/particle/momentum/class_4/event/px_bias": 0.11546026170253754, "metrics/particle/momentum/class_4/event/py_bias": -0.0020253928378224373, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.4618568420410156, "metrics/particle/momentum/class_4/event/pz_bias": 0.5097594261169434, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.029675006866455, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011109315790235996, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.03352421894669533, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.028387533500790596, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004903289023786783, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0791720524430275, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2867196500301361, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.08512484282255173, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.800241231918335, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.011882990598678589, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.046913303434848785, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04064279794692993, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008179577998816967, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12666498124599457, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.539847195148468, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.16863195598125458, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5309150218963623, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.711670994758606, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.5964994430541992, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.1856523752212524, "metrics/particle/momentum/class_5/event/px_bias": 0.022654984146356583, "metrics/particle/momentum/class_5/event/py_bias": 0.09917838871479034, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8628202080726624, "metrics/particle/momentum/class_5/event/pz_bias": -0.08842211216688156, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.68882155418396, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.2249674797058105, "step_valid_time": 1840.886579990387, "step_total_time": 1842.1115481853485, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0118233183730807, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09040112057486138, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2847, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9442786069651742, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.94240317775571, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9433399602385686, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05759682224428997, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9399668325041459, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.938099966898378, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9390324718356527, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9442786069651742, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0147452676278799, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10772638017395564, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2062, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9201065246338216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9213333333333333, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9207195203197868, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07866666666666666, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9152241455836663, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9164444444444444, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9158338885187652, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9201065246338216, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0169620862435738, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10499483935837328, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1137, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1126, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8848249027237354, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8834498834498834, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8841368584758943, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11655011655011654, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8762645914396887, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8749028749028749, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8755832037325039, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8848249027237354} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/hyperparameters.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..7f1bceb04f0802b6df7938230145787eabbe9081 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3114258, "dataset": "clic", "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": true, "use_flash_attn_varlen": true, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 128, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"clic": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["clic_edm_ttbar_pf", "clic_edm_ww_fullhad_pf", "clic_edm_qq_pf"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/model_kwargs.pkl b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..876118b75f3f236deaff12921d177959dc0271f9 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1322be402e4e0cab113d6c1b0f7376da976a5e68bae3bf8cca3c8940d9176b96 +size 4989 diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/particleflow_spec.yaml b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d68d9a47b34ab9ee2cfd53d103b7297a2e464653 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..73ad52f2444af9f6f34518aee1a66c726ace1e04 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e91e7441a32a3f93d54d154cbf1f8b14a93e5aea Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..003fd44f089e8c07943691079abfb44fb651dce4 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..79a5fa370381815fa3d9ff8266d0ebea5d885287 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..9fabf6fcf3a1c77b3612bd8754fea6f32354f2bf Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3254f9c65a1a0f7abf05f107ba7552d14ce8bdd1 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..89e2605451b882f0c65665b5f43e7251b1a38264 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..900f3e36a3507c188b0c2ba64da507cbbe156928 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..4c01ad7aa0d7acbf8f9251f4066dbe3c5391edb7 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c47b5d7745bd9412644afd92c1becfc41f886686 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..da41137f4fc67ff40e43a54c57f60c4e85c8c463 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_qq_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ed60a5e8dc13ad3dec9bc44ff8c5b573f8eece4a Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..7ecb31025f3c81babf74fd3c939b4dfd7656e620 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..14baebe4dbcd284a6a164bc3d20e51c6a61648cb Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..7e413b637edc56c785cb7c579d95ca7e4b93363c Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6cc5e2a696ae36f4102b39e361e1a156ec4007a5 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..35daf6df4e6ce85b2c50b4b5b93db23c350730dd Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25b54297b3cd38f868be262bdb255d715282e850 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..b28bbd59b21e0250516f10b95facff2cc5509dc8 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..46e644b0ac33bfb60e2176eb33481a6249fbf093 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..21754483aaaf9e1485193ec3891ee37a51811ca0 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..963a6bede99b953bae330918d3969f799276a625 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..26eb917e7232c3ee93ab70c8f967ca2199d6a33c Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ttbar_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f136fb901b359ba5815211f37d0f8848dcb7b3d4 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..2ddc71460c8459202b7619411e3c5cbb575843cd Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..75b0a95c25556bb57a89d5b11018a1e4edf52fbc Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..406490137218c95c704cc19d591cb5a19f0bcabf Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8c7dd61c852bb240de362558cc88082ded3a5e19 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..f90bbe94d8b6156ad73376906aa3792f71b89cb6 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..099f7248708289381352b88d99dba93583ccff0e Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..626d4b96037cc392fe9c375b2dc1fdcf6647fe8c Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d2d7c2e259db66799f874fba24477cec70a48d93 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..cd434e1accfc4dade9c91eb98b82283c5dd8da5e Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..246bb437d4d5d15fd90f104f698d60ba9da97a47 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b85dd7603ef820f1dd1bf9900a8e17ef651e0d78 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/train/events.out.tfevents.1793178753.nid005777.42008.0 b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/train/events.out.tfevents.1793178753.nid005777.42008.0 new file mode 100644 index 0000000000000000000000000000000000000000..b04baee187da893aac1d9d5f81c450bf3b639b30 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/train/events.out.tfevents.1793178753.nid005777.42008.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7bb93355bb59b704baea3cc38e69f8286f1d3df6a68abcd203b91b3792283db +size 15743902 diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/valid/events.out.tfevents.1793178753.nid005777.42008.1 b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/valid/events.out.tfevents.1793178753.nid005777.42008.1 new file mode 100644 index 0000000000000000000000000000000000000000..7554ba77d5d2fe86c34f12e89dd68d12f1d7fa3d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/runs/valid/events.out.tfevents.1793178753.nid005777.42008.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a7ce1242092f1babbb18646211c263f6e0ce05636187fac9b4911421de8fdc +size 28597401 diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/scenario-manifest.json b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..66c3fab75010ddbe53d31877df4e5f1ca7c20055 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/scenario-manifest.json @@ -0,0 +1,818 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239272-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239272-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "clic_pf_attention", + "model_name": "pyg-clic-v1", + "production_name": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 128, + "gpu_batch_multiplier": 128, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "seed": 12345, + "gpu_batch_multiplier": 128 + } + }, + "resolved_config": { + "dataset": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": true, + "use_flash_attn_varlen": true, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 128, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "clic": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "clic": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "clic_edm_ttbar_pf", + "clic_edm_ww_fullhad_pf", + "clic_edm_qq_pf" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 1 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--seed", + "12345", + "--gpu_batch_multiplier", + "128" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train-config.yaml b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a777e0a0db26713312a9b87c47671a2ff9f27f52 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train-config.yaml @@ -0,0 +1,278 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds +dataset: clic +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- clic_edm_ttbar_pf +- clic_edm_ww_fullhad_pf +- clic_edm_qq_pf +gpu_batch_multiplier: 128 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: true + use_jagged_attention: true + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: attention +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + clic_edm_qq_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + clic: + physical: + batch_size: 1 + samples: + clic_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + clic: + physical: + batch_size: 1 + samples: + clic_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log new file mode 100644 index 0000000000000000000000000000000000000000..445051f6fad648f33ce5f8bd86a23e1b86c29a15 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log @@ -0,0 +1,13849 @@ +rank0 [2026-10-28 11:12:24,874] INFO : Will use single-gpu: AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,875] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-10-28 11:12:24,881] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,882] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,882] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,882] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,883] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,919] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,919] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,919] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,922] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,922] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,925] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,925] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,928] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,928] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,931] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:24,931] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,934] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:24,934] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,936] INFO : Backbone initialization took 0.02s +rank0 [2026-10-28 11:12:24,937] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:24,937] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:24,941] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:24,942] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:24,942] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:24,942] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:24,942] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:24,942] INFO : classification_norm parameters: 512 +rank0 [2026-10-28 11:12:24,942] INFO : regression_norm parameters: 512 +rank0 [2026-10-28 11:12:24,942] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:24,942] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:24,943] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:24,943] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:24,943] INFO : Total MLPF parameters: 3114258 +rank0 [2026-10-28 11:12:24,944] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:24,944] INFO : Instantiated model +rank0 [2026-10-28 11:12:24,945] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,169] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,170] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,249] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +131 nn_energy.nn.1.0.bias 256 0 +132 nn_energy.nn.1.2.weight 256 0 +133 nn_energy.nn.1.2.bias 256 0 +134 nn_energy.nn.1.4.weight 256 0 +135 nn_energy.nn.1.4.bias 1 0 + +[136 rows x 3 columns] +rank0 [2026-10-28 11:12:25,249] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,251] INFO : Trainable parameters: 3114258 +rank0 [2026-10-28 11:12:25,251] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:25,251] INFO : Total parameters: 3114258 +rank0 [2026-10-28 11:12:25,252] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 +backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 +backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 +backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 +backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 +backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 +backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,256] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283 +rank0 [2026-10-28 11:12:25,256] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283 +rank0 [2026-10-28 11:12:25,256] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,256] INFO : sample=clic_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,313] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,355] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,395] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,416] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,473] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,512] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,549] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,591] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,614] INFO : train_dataset: clic_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,614] INFO : sample=clic_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,650] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,679] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,704] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,744] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,783] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,814] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,842] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,876] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,900] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,935] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,935] INFO : sample=clic_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,973] INFO : train_dataset: clic_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,012] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,082] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,102] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,123] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,172] INFO : train_dataset: clic_edm_qq_pf, 89996 +rank0 [2026-10-28 11:12:33,199] INFO : train_dataset: clic_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:33,229] INFO : train_dataset: clic_edm_qq_pf, 89994 +rank0 [2026-10-28 11:12:33,262] INFO : train_dataset: clic_edm_qq_pf, 89995 +rank0 [2026-10-28 11:12:33,277] INFO : train_dataset: clic_edm_qq_pf, 90099 +rank0 [2026-10-28 11:12:33,277] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,277] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,277] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,277] INFO : sample=clic_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,283] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,300] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,306] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,312] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,318] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,324] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,330] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : sample=clic_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,341] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,346] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,352] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,357] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,362] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,367] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,373] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,378] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,383] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : sample=clic_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,393] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,398] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,403] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,409] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,414] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,419] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,424] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,429] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,434] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,439] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,475] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,478] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,478] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:12:33,478] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,478] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,484] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,502] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,757] INFO : Created train_iterator +rank0 [2026-10-28 11:13:38,245] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 4.923548955917358, 'Classification': 0.14500824306160212, 'Regression_pt': 0.33665568113327027, 'Regression_eta': 0.1174755490757525, 'Regression_sin_phi': 0.06220133699476719, 'Regression_cos_phi': 0.09051778480410576, 'Regression_energy': 0.3657260942459106}, weights={'Classification_binary': 1.0, 'Classification': 33.95357894897461, 'Regression_pt': 14.624879837036133, 'Regression_eta': 41.9112663269043, 'Regression_sin_phi': 79.15503692626953, 'Regression_cos_phi': 54.393165588378906, 'Regression_energy': 13.462394714355469} +rank0 [2026-10-28 11:13:40,247] INFO : Step 100/50000 rank0 | Train Loss: 5.4392 | Train Loss Unweighted: 5.4392 | Train Loss Components: Classification: 0.0536 | Classification_binary: 4.6139 | Regression_cos_phi: 0.0261 | Regression_energy: 0.3386 | Regression_eta: 0.0561 | Regression_pt: 0.3237 | Regression_sin_phi: 0.0273 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 4.6139 | Regression_cos_phi: 0.0261 | Regression_energy: 0.3386 | Regression_eta: 0.0561 | Regression_pt: 0.3237 | Regression_sin_phi: 0.0273 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 2.3315s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.1049 | max_backward: 0.0335 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 1.0047 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2690.5 | uss_MB: 1982.7 | child_rss_MB: 3287.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:13:40,463] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:02,617] INFO : Step 200/50000 rank0 | Train Loss: 18.1629 | Train Loss Unweighted: 5.1827 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.4838 | Regression_cos_phi: 0.0210 | Regression_energy: 0.2930 | Regression_eta: 0.0264 | Regression_pt: 0.2925 | Regression_sin_phi: 0.0214 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.5144 | Classification_binary: 4.4838 | Regression_cos_phi: 1.1427 | Regression_energy: 3.9443 | Regression_eta: 1.1047 | Regression_pt: 4.2777 | Regression_sin_phi: 1.6953 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 1.1213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0147 | max_loss: 0.0030 | max_backward: 0.1406 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2653 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2689.8 | uss_MB: 1984.5 | child_rss_MB: 3290.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:02,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:23,600] INFO : Step 300/50000 rank0 | Train Loss: 17.0852 | Train Loss Unweighted: 5.0103 | Train Loss Components: Classification: 0.0427 | Classification_binary: 4.3273 | Regression_cos_phi: 0.0175 | Regression_energy: 0.2916 | Regression_eta: 0.0183 | Regression_pt: 0.2961 | Regression_sin_phi: 0.0169 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4486 | Classification_binary: 4.3273 | Regression_cos_phi: 0.9508 | Regression_energy: 3.9258 | Regression_eta: 0.7682 | Regression_pt: 4.3300 | Regression_sin_phi: 1.3346 | LR: 1.00e-03 | DataLoad Time: 0.0013s | Model Forward Time: 1.0807s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0029 | max_backward: 0.0340 | max_optimizer_step: 0.1593 | max_tensorboard_logging: 0.2690 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2689.7 | uss_MB: 1985.1 | child_rss_MB: 3285.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:14:23,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:43,172] INFO : Step 400/50000 rank0 | Train Loss: 15.0521 | Train Loss Unweighted: 4.8845 | Train Loss Components: Classification: 0.0439 | Classification_binary: 4.3147 | Regression_cos_phi: 0.0139 | Regression_energy: 0.2395 | Regression_eta: 0.0172 | Regression_pt: 0.2428 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4900 | Classification_binary: 4.3147 | Regression_cos_phi: 0.7560 | Regression_energy: 3.2239 | Regression_eta: 0.7198 | Regression_pt: 3.5505 | Regression_sin_phi: 0.9971 | LR: 1.00e-03 | DataLoad Time: 0.1810s | Model Forward Time: 1.0707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0156 | max_loss: 0.0029 | max_backward: 0.0821 | max_optimizer_step: 0.1022 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2691.5 | uss_MB: 1987.6 | child_rss_MB: 3283.6 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:14:43,387] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:02,383] INFO : Step 500/50000 rank0 | Train Loss: 15.7330 | Train Loss Unweighted: 4.8445 | Train Loss Components: Classification: 0.0406 | Classification_binary: 4.2336 | Regression_cos_phi: 0.0142 | Regression_energy: 0.2630 | Regression_eta: 0.0173 | Regression_pt: 0.2594 | Regression_sin_phi: 0.0163 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3787 | Classification_binary: 4.2336 | Regression_cos_phi: 0.7742 | Regression_energy: 3.5408 | Regression_eta: 0.7241 | Regression_pt: 3.7943 | Regression_sin_phi: 1.2874 | LR: 1.00e-03 | DataLoad Time: 0.0584s | Model Forward Time: 1.1432s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0495 | max_forward: 0.0136 | max_loss: 0.0028 | max_backward: 0.0327 | max_optimizer_step: 0.0972 | max_tensorboard_logging: 0.3586 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2689.1 | uss_MB: 1985.7 | child_rss_MB: 3279.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:15:02,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:21,393] INFO : Step 600/50000 rank0 | Train Loss: 16.9350 | Train Loss Unweighted: 5.0748 | Train Loss Components: Classification: 0.0427 | Classification_binary: 4.4156 | Regression_cos_phi: 0.0167 | Regression_energy: 0.2808 | Regression_eta: 0.0234 | Regression_pt: 0.2789 | Regression_sin_phi: 0.0167 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4500 | Classification_binary: 4.4156 | Regression_cos_phi: 0.9108 | Regression_energy: 3.7802 | Regression_eta: 0.9792 | Regression_pt: 4.0790 | Regression_sin_phi: 1.3202 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 1.3646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1218 | max_loss: 0.0029 | max_backward: 0.0336 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 2626.5 | uss_MB: 1922.6 | child_rss_MB: 3276.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:15:21,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:40,504] INFO : Step 700/50000 rank0 | Train Loss: 15.3225 | Train Loss Unweighted: 4.8653 | Train Loss Components: Classification: 0.0413 | Classification_binary: 4.2867 | Regression_cos_phi: 0.0158 | Regression_energy: 0.2475 | Regression_eta: 0.0162 | Regression_pt: 0.2425 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4008 | Classification_binary: 4.2867 | Regression_cos_phi: 0.8587 | Regression_energy: 3.3323 | Regression_eta: 0.6772 | Regression_pt: 3.5462 | Regression_sin_phi: 1.2206 | LR: 1.00e-03 | DataLoad Time: 0.0013s | Model Forward Time: 1.0756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.0337 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.2630 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2629.6 | uss_MB: 1926.4 | child_rss_MB: 3278.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:15:40,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:59,496] INFO : Step 800/50000 rank0 | Train Loss: 15.0226 | Train Loss Unweighted: 4.8587 | Train Loss Components: Classification: 0.0399 | Classification_binary: 4.2805 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2470 | Regression_eta: 0.0149 | Regression_pt: 0.2497 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3533 | Classification_binary: 4.2805 | Regression_cos_phi: 0.7294 | Regression_energy: 3.3248 | Regression_eta: 0.6265 | Regression_pt: 3.6512 | Regression_sin_phi: 1.0569 | LR: 9.99e-04 | DataLoad Time: 0.0179s | Model Forward Time: 1.1970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0155 | max_loss: 0.0040 | max_backward: 0.0332 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.2703 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2629.6 | uss_MB: 1927.4 | child_rss_MB: 3276.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:15:59,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:18,014] INFO : Step 900/50000 rank0 | Train Loss: 14.4804 | Train Loss Unweighted: 4.7851 | Train Loss Components: Classification: 0.0420 | Classification_binary: 4.2489 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2265 | Regression_eta: 0.0172 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4273 | Classification_binary: 4.2489 | Regression_cos_phi: 0.6153 | Regression_energy: 3.0486 | Regression_eta: 0.7199 | Regression_pt: 3.2900 | Regression_sin_phi: 1.1304 | LR: 9.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 0.9911s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0132 | max_loss: 0.0031 | max_backward: 0.0326 | max_optimizer_step: 0.0959 | max_tensorboard_logging: 0.2582 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2630.0 | uss_MB: 1929.1 | child_rss_MB: 3272.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:16:18,227] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:36,500] INFO : Step 1000/50000 rank0 | Train Loss: 14.1550 | Train Loss Unweighted: 4.7953 | Train Loss Components: Classification: 0.0389 | Classification_binary: 4.2751 | Regression_cos_phi: 0.0130 | Regression_energy: 0.2193 | Regression_eta: 0.0165 | Regression_pt: 0.2198 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 4.2751 | Regression_cos_phi: 0.7094 | Regression_energy: 2.9529 | Regression_eta: 0.6901 | Regression_pt: 3.2151 | Regression_sin_phi: 0.9906 | LR: 9.99e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1159s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.1418 | max_optimizer_step: 0.0990 | max_tensorboard_logging: 0.2606 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2630.5 | uss_MB: 1931.3 | child_rss_MB: 3267.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:16:36,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:36,743] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:16:36,824] INFO : Running validation on rank0 +rank0 [2026-10-28 11:16:36,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:16:36,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:16:38,130] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:16:38,316] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:16:38,413] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:16:38,473] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:16:38,534] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:16:38,584] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:16:38,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:16:38,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:16:38,761] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:16:38,847] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:16:38,934] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:16:38,993] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:16:39,053] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:16:39,102] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:16:39,163] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:16:39,225] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:16:39,283] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:16:39,287] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:16:39,290] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:16:39,293] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:16:39,295] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:16:39,298] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:16:39,299] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:16:39,302] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:16:39,304] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:16:39,305] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:16:39,308] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:16:39,310] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:16:39,313] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:16:39,314] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:16:39,316] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:16:39,319] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:16:44,918] INFO : VALIDATION | Step=1000/50000 | Train Loss=14.1550 | Valid Loss=14.8413 | Train Loss Unweighted=4.7953 | Valid Loss Unweighted=4.7426 | Stale=0 | ETA=205.3m +rank0 [2026-10-28 11:16:44,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:44,988] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:16:44,998] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:45,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:47,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:48,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:48,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:49,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:49,593] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:16:49,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:49,652] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:16:49,655] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:16:49,706] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:51,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:51,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:52,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:52,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:52,904] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:52,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:52,962] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:16:52,965] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:16:53,035] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:54,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:54,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:55,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:55,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:55,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:56,005] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:56,261] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:57,235] INFO : Plotted number of elements +rank0 [2026-10-28 11:16:58,689] INFO : Plotted jets +rank0 [2026-10-28 11:16:59,769] INFO : Plotted jet ratio +rank0 [2026-10-28 11:16:59,801] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:00,028] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:00,401] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:01,776] INFO : Plotted jets +rank0 [2026-10-28 11:17:02,880] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:02,913] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:17:03,112] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:17:03,830] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:05,203] INFO : Plotted jets +rank0 [2026-10-28 11:17:06,278] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:24,611] INFO : Step 1100/50000 rank0 | Train Loss: 15.1176 | Train Loss Unweighted: 4.6801 | Train Loss Components: Classification: 0.0426 | Classification_binary: 4.0828 | Regression_cos_phi: 0.0140 | Regression_energy: 0.2556 | Regression_eta: 0.0154 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4479 | Classification_binary: 4.0828 | Regression_cos_phi: 0.7597 | Regression_energy: 3.4407 | Regression_eta: 0.6470 | Regression_pt: 3.7646 | Regression_sin_phi: 0.9749 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2608 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2905.5 | uss_MB: 2392.2 | child_rss_MB: 3266.3 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:17:24,823] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:43,392] INFO : Step 1200/50000 rank0 | Train Loss: 15.1457 | Train Loss Unweighted: 4.6590 | Train Loss Components: Classification: 0.0389 | Classification_binary: 4.0706 | Regression_cos_phi: 0.0156 | Regression_energy: 0.2529 | Regression_eta: 0.0136 | Regression_pt: 0.2516 | Regression_sin_phi: 0.0158 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 4.0706 | Regression_cos_phi: 0.8468 | Regression_energy: 3.4048 | Regression_eta: 0.5712 | Regression_pt: 3.6790 | Regression_sin_phi: 1.2517 | LR: 9.99e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0417 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3703 | max_tensorboard_flush: 0.0147 | Memory: rss_MB: 2906.5 | uss_MB: 2393.1 | child_rss_MB: 3262.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:17:43,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:02,122] INFO : Step 1300/50000 rank0 | Train Loss: 15.2010 | Train Loss Unweighted: 4.7283 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.1439 | Regression_cos_phi: 0.0153 | Regression_energy: 0.2524 | Regression_eta: 0.0143 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4147 | Classification_binary: 4.1439 | Regression_cos_phi: 0.8310 | Regression_energy: 3.3984 | Regression_eta: 0.6009 | Regression_pt: 3.5873 | Regression_sin_phi: 1.2249 | LR: 9.98e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0519 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1036 | max_tensorboard_logging: 0.3592 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2907.3 | uss_MB: 2393.7 | child_rss_MB: 3262.1 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:02,334] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:20,704] INFO : Step 1400/50000 rank0 | Train Loss: 14.2392 | Train Loss Unweighted: 4.6683 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.1235 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2344 | Regression_eta: 0.0124 | Regression_pt: 0.2339 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3110 | Classification_binary: 4.1235 | Regression_cos_phi: 0.6622 | Regression_energy: 3.1555 | Regression_eta: 0.5217 | Regression_pt: 3.4212 | Regression_sin_phi: 1.0440 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1455s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.1389 | max_optimizer_step: 0.1020 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2907.3 | uss_MB: 2394.2 | child_rss_MB: 3263.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:20,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:38,932] INFO : Step 1500/50000 rank0 | Train Loss: 14.0094 | Train Loss Unweighted: 4.4789 | Train Loss Components: Classification: 0.0390 | Classification_binary: 3.9469 | Regression_cos_phi: 0.0135 | Regression_energy: 0.2247 | Regression_eta: 0.0146 | Regression_pt: 0.2271 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3239 | Classification_binary: 3.9469 | Regression_cos_phi: 0.7322 | Regression_energy: 3.0243 | Regression_eta: 0.6110 | Regression_pt: 3.3209 | Regression_sin_phi: 1.0501 | LR: 9.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1052s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0163 | max_loss: 0.0727 | max_backward: 0.0339 | max_optimizer_step: 0.0992 | max_tensorboard_logging: 0.2714 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 2908.3 | uss_MB: 2394.8 | child_rss_MB: 3289.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:39,144] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:57,280] INFO : Step 1600/50000 rank0 | Train Loss: 14.5911 | Train Loss Unweighted: 4.5934 | Train Loss Components: Classification: 0.0378 | Classification_binary: 4.0348 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2408 | Regression_eta: 0.0123 | Regression_pt: 0.2365 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2836 | Classification_binary: 4.0348 | Regression_cos_phi: 0.8999 | Regression_energy: 3.2415 | Regression_eta: 0.5171 | Regression_pt: 3.4581 | Regression_sin_phi: 1.1561 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1902s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0807 | max_loss: 0.0032 | max_backward: 0.0324 | max_optimizer_step: 0.0997 | max_tensorboard_logging: 0.3736 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2792.1 | uss_MB: 2280.0 | child_rss_MB: 3260.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:18:57,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:15,526] INFO : Step 1700/50000 rank0 | Train Loss: 13.7400 | Train Loss Unweighted: 4.4868 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.9588 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2282 | Regression_eta: 0.0106 | Regression_pt: 0.2255 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 3.9588 | Regression_cos_phi: 0.6881 | Regression_energy: 3.0726 | Regression_eta: 0.4453 | Regression_pt: 3.2978 | Regression_sin_phi: 0.9556 | LR: 9.97e-04 | DataLoad Time: 0.0337s | Model Forward Time: 1.2122s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0930 | max_forward: 0.0147 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1012 | max_tensorboard_logging: 0.3533 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2793.1 | uss_MB: 2283.0 | child_rss_MB: 3259.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:19:15,738] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:33,648] INFO : Step 1800/50000 rank0 | Train Loss: 14.2339 | Train Loss Unweighted: 4.4368 | Train Loss Components: Classification: 0.0392 | Classification_binary: 3.8847 | Regression_cos_phi: 0.0131 | Regression_energy: 0.2364 | Regression_eta: 0.0158 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3311 | Classification_binary: 3.8847 | Regression_cos_phi: 0.7107 | Regression_energy: 3.1821 | Regression_eta: 0.6614 | Regression_pt: 3.4305 | Regression_sin_phi: 1.0332 | LR: 9.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2048s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1081 | max_loss: 0.0029 | max_backward: 0.0335 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.3605 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2793.1 | uss_MB: 2283.2 | child_rss_MB: 3259.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:33,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:51,864] INFO : Step 1900/50000 rank0 | Train Loss: 14.1379 | Train Loss Unweighted: 4.5553 | Train Loss Components: Classification: 0.0416 | Classification_binary: 4.0280 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2180 | Regression_eta: 0.0125 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0141 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4130 | Classification_binary: 4.0280 | Regression_cos_phi: 0.8175 | Regression_energy: 2.9346 | Regression_eta: 0.5243 | Regression_pt: 3.3063 | Regression_sin_phi: 1.1142 | LR: 9.97e-04 | DataLoad Time: 0.1309s | Model Forward Time: 1.2568s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1231 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1072 | max_tensorboard_logging: 0.3578 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2794.1 | uss_MB: 2283.7 | child_rss_MB: 3256.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:52,079] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:10,326] INFO : Step 2000/50000 rank0 | Train Loss: 13.6430 | Train Loss Unweighted: 4.4829 | Train Loss Components: Classification: 0.0374 | Classification_binary: 3.9654 | Regression_cos_phi: 0.0119 | Regression_energy: 0.2202 | Regression_eta: 0.0164 | Regression_pt: 0.2205 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2688 | Classification_binary: 3.9654 | Regression_cos_phi: 0.6460 | Regression_energy: 2.9643 | Regression_eta: 0.6872 | Regression_pt: 3.2254 | Regression_sin_phi: 0.8858 | LR: 9.96e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2159s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0833 | max_forward: 0.0164 | max_loss: 0.0029 | max_backward: 0.0342 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.3554 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 2794.1 | uss_MB: 2284.5 | child_rss_MB: 3257.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:20:10,541] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:10,558] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:20:10,742] INFO : Running validation on rank0 +rank0 [2026-10-28 11:20:10,742] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:20:10,742] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:20:11,823] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:20:11,966] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:20:12,095] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:20:12,169] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:20:12,237] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:20:12,304] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:20:12,373] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:20:12,441] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:20:12,544] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:20:12,667] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:20:12,789] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:20:12,861] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:20:12,930] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:20:12,998] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:20:13,067] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:20:13,136] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:20:13,248] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:20:13,255] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:20:13,258] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:20:13,261] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:20:13,263] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:20:13,265] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:20:13,268] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:20:13,270] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:20:13,271] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:20:13,274] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:20:13,277] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:20:13,279] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:20:13,282] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:20:13,284] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:20:13,286] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:20:13,289] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:19,018] INFO : VALIDATION | Step=2000/50000 | Train Loss=13.6430 | Valid Loss=14.0077 | Train Loss Unweighted=4.4829 | Valid Loss Unweighted=4.5018 | Stale=0 | ETA=186.2m +rank0 [2026-10-28 11:20:19,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:19,093] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:19,097] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,239] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:20,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:21,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:21,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:22,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:22,468] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:22,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:22,528] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:22,531] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:22,670] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:24,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:24,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:25,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:25,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:25,867] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:25,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:25,924] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:25,927] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:20:26,001] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:27,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:27,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:28,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:28,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:28,953] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:28,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:29,212] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:29,662] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:31,087] INFO : Plotted jets +rank0 [2026-10-28 11:20:32,164] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:32,197] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:32,422] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:32,795] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:34,583] INFO : Plotted jets +rank0 [2026-10-28 11:20:35,687] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:35,721] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:35,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:36,333] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:37,714] INFO : Plotted jets +rank0 [2026-10-28 11:20:38,802] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:57,150] INFO : Step 2100/50000 rank0 | Train Loss: 13.9090 | Train Loss Unweighted: 4.4638 | Train Loss Components: Classification: 0.0408 | Classification_binary: 3.9347 | Regression_cos_phi: 0.0123 | Regression_energy: 0.2244 | Regression_eta: 0.0132 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3861 | Classification_binary: 3.9347 | Regression_cos_phi: 0.6698 | Regression_energy: 3.0209 | Regression_eta: 0.5551 | Regression_pt: 3.2908 | Regression_sin_phi: 1.0515 | LR: 9.96e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0340 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.3716 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 3010.8 | uss_MB: 2501.0 | child_rss_MB: 3248.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:20:57,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:15,235] INFO : Step 2200/50000 rank0 | Train Loss: 13.2996 | Train Loss Unweighted: 4.3393 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.8320 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2164 | Regression_eta: 0.0152 | Regression_pt: 0.2169 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1964 | Classification_binary: 3.8320 | Regression_cos_phi: 0.6959 | Regression_energy: 2.9132 | Regression_eta: 0.6360 | Regression_pt: 3.1722 | Regression_sin_phi: 0.8538 | LR: 9.96e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2057s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0626 | max_loss: 0.0031 | max_backward: 0.0384 | max_optimizer_step: 0.1033 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3010.8 | uss_MB: 2503.4 | child_rss_MB: 3246.6 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:21:15,449] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:33,136] INFO : Step 2300/50000 rank0 | Train Loss: 13.9989 | Train Loss Unweighted: 4.5055 | Train Loss Components: Classification: 0.0379 | Classification_binary: 3.9602 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2362 | Regression_eta: 0.0131 | Regression_pt: 0.2344 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2861 | Classification_binary: 3.9602 | Regression_cos_phi: 0.6195 | Regression_energy: 3.1793 | Regression_eta: 0.5504 | Regression_pt: 3.4285 | Regression_sin_phi: 0.9748 | LR: 9.95e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0157 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.2054 | max_tensorboard_logging: 0.2590 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3010.8 | uss_MB: 2501.8 | child_rss_MB: 3246.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:21:33,360] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:50,958] INFO : Step 2400/50000 rank0 | Train Loss: 13.4525 | Train Loss Unweighted: 4.4140 | Train Loss Components: Classification: 0.0364 | Classification_binary: 3.9141 | Regression_cos_phi: 0.0124 | Regression_energy: 0.2130 | Regression_eta: 0.0150 | Regression_pt: 0.2097 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2375 | Classification_binary: 3.9141 | Regression_cos_phi: 0.6756 | Regression_energy: 2.8670 | Regression_eta: 0.6268 | Regression_pt: 3.0675 | Regression_sin_phi: 1.0641 | LR: 9.95e-04 | DataLoad Time: 0.0535s | Model Forward Time: 1.2218s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0857 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3604 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 3010.8 | uss_MB: 2502.2 | child_rss_MB: 3244.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:21:51,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:08,848] INFO : Step 2500/50000 rank0 | Train Loss: 13.2218 | Train Loss Unweighted: 4.4375 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.9428 | Regression_cos_phi: 0.0124 | Regression_energy: 0.2096 | Regression_eta: 0.0123 | Regression_pt: 0.2115 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2727 | Classification_binary: 3.9428 | Regression_cos_phi: 0.6727 | Regression_energy: 2.8215 | Regression_eta: 0.5153 | Regression_pt: 3.0934 | Regression_sin_phi: 0.9034 | LR: 9.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1525s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.1375 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2596 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 3010.8 | uss_MB: 2502.4 | child_rss_MB: 3242.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:09,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:26,376] INFO : Step 2600/50000 rank0 | Train Loss: 13.9761 | Train Loss Unweighted: 4.4610 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.9250 | Regression_cos_phi: 0.0144 | Regression_energy: 0.2290 | Regression_eta: 0.0142 | Regression_pt: 0.2317 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1249 | Classification_binary: 3.9250 | Regression_cos_phi: 0.7806 | Regression_energy: 3.0835 | Regression_eta: 0.5940 | Regression_pt: 3.3879 | Regression_sin_phi: 1.0804 | LR: 9.94e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0030 | max_backward: 0.1410 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2613 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 3010.8 | uss_MB: 2502.8 | child_rss_MB: 3240.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:22:26,604] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:43,959] INFO : Step 2700/50000 rank0 | Train Loss: 14.3597 | Train Loss Unweighted: 4.5058 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.9467 | Regression_cos_phi: 0.0145 | Regression_energy: 0.2388 | Regression_eta: 0.0140 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2757 | Classification_binary: 3.9467 | Regression_cos_phi: 0.7873 | Regression_energy: 3.2144 | Regression_eta: 0.5887 | Regression_pt: 3.5306 | Regression_sin_phi: 1.0163 | LR: 9.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.1675s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0148 | max_loss: 0.0029 | max_backward: 0.1167 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2996 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2874.9 | uss_MB: 2366.1 | child_rss_MB: 3243.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:22:44,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:01,655] INFO : Step 2800/50000 rank0 | Train Loss: 13.2675 | Train Loss Unweighted: 4.3890 | Train Loss Components: Classification: 0.0363 | Classification_binary: 3.9086 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2006 | Regression_eta: 0.0170 | Regression_pt: 0.1999 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2324 | Classification_binary: 3.9086 | Regression_cos_phi: 0.7016 | Regression_energy: 2.7004 | Regression_eta: 0.7121 | Regression_pt: 2.9239 | Regression_sin_phi: 1.0885 | LR: 9.93e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0409 | max_backward: 0.0336 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2874.9 | uss_MB: 2365.5 | child_rss_MB: 3266.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:01,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:19,237] INFO : Step 2900/50000 rank0 | Train Loss: 13.2001 | Train Loss Unweighted: 4.2595 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2086 | Regression_eta: 0.0160 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2450 | Classification_binary: 3.7678 | Regression_cos_phi: 0.6078 | Regression_energy: 2.8076 | Regression_eta: 0.6689 | Regression_pt: 3.0049 | Regression_sin_phi: 1.0982 | LR: 9.93e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1964s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0503 | max_loss: 0.0029 | max_backward: 0.0335 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 2875.9 | uss_MB: 2368.2 | child_rss_MB: 3238.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:23:19,452] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:36,846] INFO : Step 3000/50000 rank0 | Train Loss: 12.9697 | Train Loss Unweighted: 4.3787 | Train Loss Components: Classification: 0.0363 | Classification_binary: 3.9002 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2025 | Regression_eta: 0.0130 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2330 | Classification_binary: 3.9002 | Regression_cos_phi: 0.6999 | Regression_energy: 2.7262 | Regression_eta: 0.5441 | Regression_pt: 2.9601 | Regression_sin_phi: 0.9062 | LR: 9.92e-04 | DataLoad Time: 0.1146s | Model Forward Time: 1.2229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0457 | max_forward: 0.0156 | max_loss: 0.0029 | max_backward: 0.0333 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2875.9 | uss_MB: 2368.7 | child_rss_MB: 3238.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:23:37,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:37,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:23:37,231] INFO : Running validation on rank0 +rank0 [2026-10-28 11:23:37,231] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:23:37,231] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:23:38,304] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:23:38,446] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:23:38,576] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:23:38,649] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:23:38,717] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:23:38,783] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:23:38,852] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:23:38,920] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:23:39,024] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:23:39,145] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:23:39,266] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:23:39,337] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:23:39,405] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:23:39,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:23:39,541] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:23:39,610] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:23:39,721] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:23:39,728] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:23:39,730] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:23:39,733] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:23:39,736] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:23:39,737] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:23:39,740] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:23:39,742] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:23:39,743] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:23:39,746] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:23:39,748] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:23:39,749] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:23:39,752] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:23:39,755] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:23:39,757] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:23:39,758] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:23:45,623] INFO : VALIDATION | Step=3000/50000 | Train Loss=12.9697 | Valid Loss=13.2191 | Train Loss Unweighted=4.3787 | Valid Loss Unweighted=4.3149 | Stale=0 | ETA=175.5m +rank0 [2026-10-28 11:23:45,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:45,693] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:23:45,697] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:45,839] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:47,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:48,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:48,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:49,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:49,243] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:23:49,247] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:49,302] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:23:49,304] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:49,380] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:50,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:51,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:51,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:52,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:52,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:52,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:52,686] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:23:52,689] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:23:52,761] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:54,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:54,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:55,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:55,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:55,751] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:55,759] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:56,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:56,461] INFO : Plotted number of elements +rank0 [2026-10-28 11:23:57,889] INFO : Plotted jets +rank0 [2026-10-28 11:23:58,970] INFO : Plotted jet ratio +rank0 [2026-10-28 11:23:59,004] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:59,230] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:59,603] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:01,358] INFO : Plotted jets +rank0 [2026-10-28 11:24:02,470] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:02,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:24:02,702] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:24:03,116] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:04,496] INFO : Plotted jets +rank0 [2026-10-28 11:24:05,589] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:23,304] INFO : Step 3100/50000 rank0 | Train Loss: 13.5068 | Train Loss Unweighted: 4.3736 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.8624 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2195 | Regression_eta: 0.0127 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1654 | Classification_binary: 3.8624 | Regression_cos_phi: 0.6915 | Regression_energy: 2.9554 | Regression_eta: 0.5329 | Regression_pt: 3.1873 | Regression_sin_phi: 1.1120 | LR: 9.91e-04 | DataLoad Time: 0.1919s | Model Forward Time: 1.0768s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3089.1 | uss_MB: 2580.5 | child_rss_MB: 3237.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:24:23,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:40,840] INFO : Step 3200/50000 rank0 | Train Loss: 12.9836 | Train Loss Unweighted: 4.3833 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.9012 | Regression_cos_phi: 0.0115 | Regression_energy: 0.2042 | Regression_eta: 0.0154 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1402 | Classification_binary: 3.9012 | Regression_cos_phi: 0.6238 | Regression_energy: 2.7494 | Regression_eta: 0.6460 | Regression_pt: 3.0101 | Regression_sin_phi: 0.9128 | LR: 9.91e-04 | DataLoad Time: 0.1708s | Model Forward Time: 1.1519s | Phase Times: max_pre_step_barrier: 0.0119 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.3147 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3089.1 | uss_MB: 2580.8 | child_rss_MB: 3236.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:24:41,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:58,359] INFO : Step 3300/50000 rank0 | Train Loss: 13.2314 | Train Loss Unweighted: 4.2246 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.7231 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2121 | Regression_eta: 0.0141 | Regression_pt: 0.2151 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1837 | Classification_binary: 3.7231 | Regression_cos_phi: 0.6084 | Regression_energy: 2.8552 | Regression_eta: 0.5891 | Regression_pt: 3.1455 | Regression_sin_phi: 1.1264 | LR: 9.90e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0888s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3089.1 | uss_MB: 2581.1 | child_rss_MB: 3234.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:24:58,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:15,901] INFO : Step 3400/50000 rank0 | Train Loss: 12.4928 | Train Loss Unweighted: 4.1350 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.6646 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1999 | Regression_eta: 0.0121 | Regression_pt: 0.2027 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0997 | Classification_binary: 3.6646 | Regression_cos_phi: 0.6160 | Regression_energy: 2.6907 | Regression_eta: 0.5090 | Regression_pt: 2.9646 | Regression_sin_phi: 0.9481 | LR: 9.90e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.0837s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3089.1 | uss_MB: 2581.4 | child_rss_MB: 3233.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:25:16,117] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:33,436] INFO : Step 3500/50000 rank0 | Train Loss: 12.7254 | Train Loss Unweighted: 4.3004 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.8316 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1973 | Regression_eta: 0.0142 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1242 | Classification_binary: 3.8316 | Regression_cos_phi: 0.6718 | Regression_energy: 2.6556 | Regression_eta: 0.5968 | Regression_pt: 2.9298 | Regression_sin_phi: 0.9156 | LR: 9.89e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0031 | max_backward: 0.0374 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3089.1 | uss_MB: 2583.7 | child_rss_MB: 3233.7 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:25:33,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:50,865] INFO : Step 3600/50000 rank0 | Train Loss: 11.4558 | Train Loss Unweighted: 3.9913 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.5610 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1845 | Regression_eta: 0.0094 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1008 | Classification_binary: 3.5610 | Regression_cos_phi: 0.5221 | Regression_energy: 2.4843 | Regression_eta: 0.3944 | Regression_pt: 2.7168 | Regression_sin_phi: 0.6764 | LR: 9.89e-04 | DataLoad Time: 0.0885s | Model Forward Time: 1.3003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1046 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3089.1 | uss_MB: 2581.7 | child_rss_MB: 3228.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:51,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:08,266] INFO : Step 3700/50000 rank0 | Train Loss: 11.7007 | Train Loss Unweighted: 4.1144 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.6838 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1846 | Regression_eta: 0.0103 | Regression_pt: 0.1836 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1114 | Classification_binary: 3.6838 | Regression_cos_phi: 0.5157 | Regression_energy: 2.4852 | Regression_eta: 0.4335 | Regression_pt: 2.6848 | Regression_sin_phi: 0.7863 | LR: 9.88e-04 | DataLoad Time: 0.0533s | Model Forward Time: 1.2210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0565 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1051 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2953.1 | uss_MB: 2445.8 | child_rss_MB: 3233.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:26:08,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:25,777] INFO : Step 3800/50000 rank0 | Train Loss: 12.2407 | Train Loss Unweighted: 4.1208 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.6893 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1804 | Regression_eta: 0.0153 | Regression_pt: 0.1781 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0763 | Classification_binary: 3.6893 | Regression_cos_phi: 0.5752 | Regression_energy: 2.4289 | Regression_eta: 0.6416 | Regression_pt: 2.6044 | Regression_sin_phi: 1.2252 | LR: 9.87e-04 | DataLoad Time: 0.1080s | Model Forward Time: 1.2511s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0893 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1042 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2953.1 | uss_MB: 2446.4 | child_rss_MB: 3232.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:26:25,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:43,285] INFO : Step 3900/50000 rank0 | Train Loss: 13.7422 | Train Loss Unweighted: 4.0877 | Train Loss Components: Classification: 0.0348 | Classification_binary: 3.5466 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2271 | Regression_eta: 0.0269 | Regression_pt: 0.2304 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1826 | Classification_binary: 3.5466 | Regression_cos_phi: 0.6043 | Regression_energy: 3.0574 | Regression_eta: 1.1283 | Regression_pt: 3.3695 | Regression_sin_phi: 0.8536 | LR: 9.87e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0532 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2953.1 | uss_MB: 2449.3 | child_rss_MB: 3234.1 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 11:26:43,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:00,788] INFO : Step 4000/50000 rank0 | Train Loss: 13.3953 | Train Loss Unweighted: 4.2353 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0133 | Regression_energy: 0.2176 | Regression_eta: 0.0185 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1787 | Classification_binary: 3.7303 | Regression_cos_phi: 0.7229 | Regression_energy: 2.9300 | Regression_eta: 0.7735 | Regression_pt: 3.0411 | Regression_sin_phi: 1.0188 | LR: 9.86e-04 | DataLoad Time: 0.0335s | Model Forward Time: 1.2911s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1222 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 2954.1 | uss_MB: 2447.8 | child_rss_MB: 3234.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:01,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:01,021] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:27:01,208] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:01,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:01,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:02,268] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:27:02,410] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:02,541] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:02,614] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:02,682] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:02,749] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:02,819] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:02,888] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:02,996] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:03,117] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:03,243] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:03,313] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:03,381] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:03,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:03,517] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:03,586] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:03,702] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:03,710] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:03,712] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:03,715] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:03,716] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:03,719] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:03,721] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:03,724] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:03,725] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:03,728] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:03,728] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:03,729] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:03,732] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:03,735] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:03,736] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:03,738] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:09,538] INFO : VALIDATION | Step=4000/50000 | Train Loss=13.3953 | Valid Loss=12.3813 | Train Loss Unweighted=4.2353 | Valid Loss Unweighted=4.1399 | Stale=0 | ETA=167.9m +rank0 [2026-10-28 11:27:09,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:09,609] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:09,612] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:09,758] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:11,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:11,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:12,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:12,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:12,945] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:12,947] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:13,003] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:13,006] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:13,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:14,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:15,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:15,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:16,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:16,231] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:16,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:16,288] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:16,290] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:27:16,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:17,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:18,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:18,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:19,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:19,486] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:19,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:19,742] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:20,193] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:21,620] INFO : Plotted jets +rank0 [2026-10-28 11:27:22,702] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:22,735] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:22,956] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:23,348] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:25,136] INFO : Plotted jets +rank0 [2026-10-28 11:27:26,247] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:26,279] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:26,480] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:26,892] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:28,275] INFO : Plotted jets +rank0 [2026-10-28 11:27:29,369] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:46,648] INFO : Step 4100/50000 rank0 | Train Loss: 12.0936 | Train Loss Unweighted: 4.1041 | Train Loss Components: Classification: 0.0330 | Classification_binary: 3.6632 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1863 | Regression_eta: 0.0137 | Regression_pt: 0.1851 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1200 | Classification_binary: 3.6632 | Regression_cos_phi: 0.6062 | Regression_energy: 2.5087 | Regression_eta: 0.5755 | Regression_pt: 2.7072 | Regression_sin_phi: 0.9128 | LR: 9.85e-04 | DataLoad Time: 0.0292s | Model Forward Time: 1.1405s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1657 | max_tensorboard_logging: 0.2765 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3125.9 | uss_MB: 2621.2 | child_rss_MB: 3232.0 | allocated_MB: 207.0 | reserved_MB: 2620.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:46,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:03,923] INFO : Step 4200/50000 rank0 | Train Loss: 11.6792 | Train Loss Unweighted: 4.1039 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.6639 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1896 | Regression_eta: 0.0104 | Regression_pt: 0.1900 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1176 | Classification_binary: 3.6639 | Regression_cos_phi: 0.4762 | Regression_energy: 2.5530 | Regression_eta: 0.4342 | Regression_pt: 2.7786 | Regression_sin_phi: 0.6556 | LR: 9.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1032s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3126.9 | uss_MB: 2621.5 | child_rss_MB: 3230.1 | allocated_MB: 207.0 | reserved_MB: 2620.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:04,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:20,968] INFO : Step 4300/50000 rank0 | Train Loss: 11.3414 | Train Loss Unweighted: 3.8949 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0121 | Regression_energy: 0.1687 | Regression_eta: 0.0126 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0356 | Classification_binary: 3.4934 | Regression_cos_phi: 0.6588 | Regression_energy: 2.2711 | Regression_eta: 0.5283 | Regression_pt: 2.4247 | Regression_sin_phi: 0.9295 | LR: 9.84e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.1365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3126.9 | uss_MB: 2621.9 | child_rss_MB: 3227.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:28:21,192] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:38,171] INFO : Step 4400/50000 rank0 | Train Loss: 11.1695 | Train Loss Unweighted: 4.0353 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.6355 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1709 | Regression_eta: 0.0119 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0032 | Classification_binary: 3.6355 | Regression_cos_phi: 0.5771 | Regression_energy: 2.3007 | Regression_eta: 0.4990 | Regression_pt: 2.4582 | Regression_sin_phi: 0.6958 | LR: 9.83e-04 | DataLoad Time: 0.1882s | Model Forward Time: 1.1807s | Phase Times: max_pre_step_barrier: 0.0684 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.3059 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3126.9 | uss_MB: 2622.2 | child_rss_MB: 3226.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:38,384] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:55,260] INFO : Step 4500/50000 rank0 | Train Loss: 12.5243 | Train Loss Unweighted: 4.1024 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.6156 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2098 | Regression_eta: 0.0112 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1611 | Classification_binary: 3.6156 | Regression_cos_phi: 0.5788 | Regression_energy: 2.8249 | Regression_eta: 0.4714 | Regression_pt: 3.0859 | Regression_sin_phi: 0.7867 | LR: 9.82e-04 | DataLoad Time: 0.0149s | Model Forward Time: 1.1967s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3126.9 | uss_MB: 2622.4 | child_rss_MB: 3225.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:55,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:12,254] INFO : Step 4600/50000 rank0 | Train Loss: 13.1041 | Train Loss Unweighted: 4.1161 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.6132 | Regression_cos_phi: 0.0120 | Regression_energy: 0.2134 | Regression_eta: 0.0141 | Regression_pt: 0.2164 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1336 | Classification_binary: 3.6132 | Regression_cos_phi: 0.6547 | Regression_energy: 2.8726 | Regression_eta: 0.5922 | Regression_pt: 3.1648 | Regression_sin_phi: 1.0730 | LR: 9.81e-04 | DataLoad Time: 0.0899s | Model Forward Time: 1.2683s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0175 | max_loss: 0.0866 | max_backward: 0.0380 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3127.9 | uss_MB: 2623.6 | child_rss_MB: 3224.7 | allocated_MB: 210.7 | reserved_MB: 2740.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:29:12,469] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:29,298] INFO : Step 4700/50000 rank0 | Train Loss: 11.6841 | Train Loss Unweighted: 3.9829 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1834 | Regression_eta: 0.0119 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0671 | Classification_binary: 3.5536 | Regression_cos_phi: 0.5649 | Regression_energy: 2.4691 | Regression_eta: 0.4994 | Regression_pt: 2.6481 | Regression_sin_phi: 0.8819 | LR: 9.81e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1011 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3027.8 | uss_MB: 2523.0 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:29,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:46,072] INFO : Step 4800/50000 rank0 | Train Loss: 11.3393 | Train Loss Unweighted: 3.9175 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.5087 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1710 | Regression_eta: 0.0120 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0445 | Classification_binary: 3.5087 | Regression_cos_phi: 0.6225 | Regression_energy: 2.3015 | Regression_eta: 0.5011 | Regression_pt: 2.5317 | Regression_sin_phi: 0.8293 | LR: 9.80e-04 | DataLoad Time: 0.0099s | Model Forward Time: 1.0692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0132 | max_loss: 0.0029 | max_backward: 0.0329 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3029.8 | uss_MB: 2525.0 | child_rss_MB: 3223.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:46,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:03,452] INFO : Step 4900/50000 rank0 | Train Loss: 11.1474 | Train Loss Unweighted: 3.8611 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.4561 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1717 | Regression_eta: 0.0105 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0501 | Classification_binary: 3.4561 | Regression_cos_phi: 0.5447 | Regression_energy: 2.3114 | Regression_eta: 0.4396 | Regression_pt: 2.5045 | Regression_sin_phi: 0.8410 | LR: 9.79e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2750s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1131 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3029.8 | uss_MB: 2525.5 | child_rss_MB: 3219.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:03,665] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:20,636] INFO : Step 5000/50000 rank0 | Train Loss: 12.4812 | Train Loss Unweighted: 4.0496 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.5790 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2003 | Regression_eta: 0.0164 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0789 | Classification_binary: 3.5790 | Regression_cos_phi: 0.6195 | Regression_energy: 2.6961 | Regression_eta: 0.6876 | Regression_pt: 2.9157 | Regression_sin_phi: 0.9045 | LR: 9.78e-04 | DataLoad Time: 0.0624s | Model Forward Time: 1.1579s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0029 | max_backward: 0.0328 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3029.8 | uss_MB: 2525.9 | child_rss_MB: 3221.2 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:20,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:20,880] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:30:20,962] INFO : Running validation on rank0 +rank0 [2026-10-28 11:30:20,962] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:30:20,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:30:22,145] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:30:22,288] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:30:22,418] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:30:22,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:30:22,558] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:30:22,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:30:22,694] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:30:22,762] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:30:22,871] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:30:22,995] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:30:23,116] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:30:23,187] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:30:23,256] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:30:23,323] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:30:23,392] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:30:23,460] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:30:23,573] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:30:23,581] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:30:23,583] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:30:23,586] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:30:23,587] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:30:23,590] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:30:23,593] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:30:23,594] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:30:23,596] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:30:23,599] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:30:23,600] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:30:23,602] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:30:23,603] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:30:23,606] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:30:23,607] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:30:23,610] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:30:29,498] INFO : VALIDATION | Step=5000/50000 | Train Loss=12.4812 | Valid Loss=11.7654 | Train Loss Unweighted=4.0496 | Valid Loss Unweighted=3.9604 | Stale=0 | ETA=161.4m +rank0 [2026-10-28 11:30:29,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:29,570] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:30:29,574] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:29,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:31,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:31,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:32,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:32,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:32,937] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:32,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:32,996] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:30:32,999] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:33,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:34,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:34,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:35,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:36,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:36,085] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:36,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:36,142] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:30:36,145] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:30:36,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:37,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:38,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:38,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:39,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:39,216] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:39,223] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:39,471] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:39,924] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:41,354] INFO : Plotted jets +rank0 [2026-10-28 11:30:42,433] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:42,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:42,691] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:43,064] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:44,831] INFO : Plotted jets +rank0 [2026-10-28 11:30:45,942] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:45,975] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:30:46,175] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:30:46,586] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:47,965] INFO : Plotted jets +rank0 [2026-10-28 11:30:49,050] INFO : Plotted jet ratio +rank0 [2026-10-28 11:31:06,102] INFO : Step 5100/50000 rank0 | Train Loss: 11.7930 | Train Loss Unweighted: 3.8677 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.4254 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1892 | Regression_eta: 0.0113 | Regression_pt: 0.1888 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9747 | Classification_binary: 3.4254 | Regression_cos_phi: 0.6876 | Regression_energy: 2.5471 | Regression_eta: 0.4726 | Regression_pt: 2.7613 | Regression_sin_phi: 0.9243 | LR: 9.77e-04 | DataLoad Time: 0.0134s | Model Forward Time: 1.1108s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1087 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3098.9 | uss_MB: 2598.3 | child_rss_MB: 3224.5 | allocated_MB: 207.0 | reserved_MB: 2600.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:06,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:22,854] INFO : Step 5200/50000 rank0 | Train Loss: 11.8786 | Train Loss Unweighted: 3.9343 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.4876 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1910 | Regression_eta: 0.0121 | Regression_pt: 0.1903 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0691 | Classification_binary: 3.4876 | Regression_cos_phi: 0.5862 | Regression_energy: 2.5708 | Regression_eta: 0.5071 | Regression_pt: 2.7826 | Regression_sin_phi: 0.8752 | LR: 9.76e-04 | DataLoad Time: 0.0397s | Model Forward Time: 1.1914s | Phase Times: max_pre_step_barrier: 0.0680 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.3068 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3100.9 | uss_MB: 2600.1 | child_rss_MB: 3223.5 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:31:23,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:39,647] INFO : Step 5300/50000 rank0 | Train Loss: 10.6609 | Train Loss Unweighted: 3.8564 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.4696 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1660 | Regression_eta: 0.0102 | Regression_pt: 0.1656 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9370 | Classification_binary: 3.4696 | Regression_cos_phi: 0.4540 | Regression_energy: 2.2341 | Regression_eta: 0.4273 | Regression_pt: 2.4223 | Regression_sin_phi: 0.7166 | LR: 9.75e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.0593s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0030 | max_backward: 0.0325 | max_optimizer_step: 0.1017 | max_tensorboard_logging: 0.2745 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3100.9 | uss_MB: 2600.6 | child_rss_MB: 3223.9 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:39,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:56,481] INFO : Step 5400/50000 rank0 | Train Loss: 11.6733 | Train Loss Unweighted: 3.9370 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.5066 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1838 | Regression_eta: 0.0129 | Regression_pt: 0.1812 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0224 | Classification_binary: 3.5066 | Regression_cos_phi: 0.6339 | Regression_energy: 2.4741 | Regression_eta: 0.5415 | Regression_pt: 2.6499 | Regression_sin_phi: 0.8450 | LR: 9.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.1020s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1072 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3100.9 | uss_MB: 2601.3 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:31:56,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:13,173] INFO : Step 5500/50000 rank0 | Train Loss: 11.6202 | Train Loss Unweighted: 3.7524 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.3283 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1758 | Regression_eta: 0.0188 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0647 | Classification_binary: 3.3283 | Regression_cos_phi: 0.6411 | Regression_energy: 2.3669 | Regression_eta: 0.7863 | Regression_pt: 2.5655 | Regression_sin_phi: 0.8676 | LR: 9.73e-04 | DataLoad Time: 0.0211s | Model Forward Time: 1.2304s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0520 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.3739 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3100.9 | uss_MB: 2601.5 | child_rss_MB: 3224.1 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:32:13,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:29,921] INFO : Step 5600/50000 rank0 | Train Loss: 11.1195 | Train Loss Unweighted: 3.8400 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.4180 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1838 | Regression_eta: 0.0118 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8854 | Classification_binary: 3.4180 | Regression_cos_phi: 0.5596 | Regression_energy: 2.4741 | Regression_eta: 0.4957 | Regression_pt: 2.6654 | Regression_sin_phi: 0.6213 | LR: 9.72e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2505s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0872 | max_loss: 0.0031 | max_backward: 0.0342 | max_optimizer_step: 0.1035 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3100.9 | uss_MB: 2601.7 | child_rss_MB: 3221.3 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:32:30,134] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:47,036] INFO : Step 5700/50000 rank0 | Train Loss: 11.1942 | Train Loss Unweighted: 3.8053 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3874 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1814 | Regression_eta: 0.0139 | Regression_pt: 0.1760 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9185 | Classification_binary: 3.3874 | Regression_cos_phi: 0.5778 | Regression_energy: 2.4424 | Regression_eta: 0.5820 | Regression_pt: 2.5734 | Regression_sin_phi: 0.7126 | LR: 9.71e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2230s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3099.1 | uss_MB: 2599.1 | child_rss_MB: 3220.0 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:47,263] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:04,165] INFO : Step 5800/50000 rank0 | Train Loss: 11.4069 | Train Loss Unweighted: 3.7753 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.3449 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1869 | Regression_eta: 0.0111 | Regression_pt: 0.1835 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9090 | Classification_binary: 3.3449 | Regression_cos_phi: 0.5895 | Regression_energy: 2.5159 | Regression_eta: 0.4634 | Regression_pt: 2.6830 | Regression_sin_phi: 0.9011 | LR: 9.70e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3101.1 | uss_MB: 2601.1 | child_rss_MB: 3219.7 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:33:04,376] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:33:20,603] INFO : Step 5900/50000 rank0 | Train Loss: 11.5323 | Train Loss Unweighted: 3.7381 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.3077 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1820 | Regression_eta: 0.0130 | Regression_pt: 0.1805 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1055 | Classification_binary: 3.3077 | Regression_cos_phi: 0.6180 | Regression_energy: 2.4508 | Regression_eta: 0.5432 | Regression_pt: 2.6391 | Regression_sin_phi: 0.8680 | LR: 9.69e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1670s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0915 | max_backward: 0.0342 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3101.1 | uss_MB: 2601.4 | child_rss_MB: 3244.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:33:20,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:37,279] INFO : Step 6000/50000 rank0 | Train Loss: 11.3068 | Train Loss Unweighted: 3.7757 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.3496 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1832 | Regression_eta: 0.0099 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 3.3496 | Regression_cos_phi: 0.6457 | Regression_energy: 2.4666 | Regression_eta: 0.4157 | Regression_pt: 2.6638 | Regression_sin_phi: 0.7739 | LR: 9.68e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1259s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0031 | max_backward: 0.0710 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3102.1 | uss_MB: 2602.1 | child_rss_MB: 3217.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:33:37,489] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:37,508] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:33:37,589] INFO : Running validation on rank0 +rank0 [2026-10-28 11:33:37,589] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:33:37,589] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:33:38,759] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:33:38,902] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:33:39,032] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:33:39,105] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:33:39,172] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:33:39,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:33:39,307] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:33:39,375] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:33:39,485] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:33:39,609] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:33:39,730] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:33:39,802] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:33:39,870] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:33:39,937] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:33:40,006] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:33:40,074] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:33:40,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:33:40,201] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:33:40,203] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:33:40,205] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:33:40,207] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:33:40,210] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:33:40,211] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:33:40,214] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:33:40,216] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:33:40,218] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:33:40,220] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:33:40,221] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:33:40,224] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:33:40,226] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:33:40,229] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:33:40,230] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:33:45,982] INFO : VALIDATION | Step=6000/50000 | Train Loss=11.3068 | Valid Loss=11.3637 | Train Loss Unweighted=3.7757 | Valid Loss Unweighted=3.8073 | Stale=0 | ETA=155.5m +rank0 [2026-10-28 11:33:45,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:46,052] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:33:46,056] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:46,202] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:47,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:48,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:48,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:49,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:49,289] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:49,293] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:49,349] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:33:49,352] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:33:49,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:50,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:51,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:51,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:52,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:52,625] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:52,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:52,683] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:33:52,686] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:33:52,747] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:54,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:54,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:55,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:55,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:55,761] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:55,768] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:56,020] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:56,472] INFO : Plotted number of elements +rank0 [2026-10-28 11:33:57,903] INFO : Plotted jets +rank0 [2026-10-28 11:33:58,986] INFO : Plotted jet ratio +rank0 [2026-10-28 11:33:59,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:33:59,240] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:00,038] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:01,424] INFO : Plotted jets +rank0 [2026-10-28 11:34:02,535] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:02,567] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:02,769] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:03,182] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:04,577] INFO : Plotted jets +rank0 [2026-10-28 11:34:05,671] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:22,354] INFO : Step 6100/50000 rank0 | Train Loss: 11.0077 | Train Loss Unweighted: 3.7377 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.3377 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1681 | Regression_eta: 0.0111 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9584 | Classification_binary: 3.3377 | Regression_cos_phi: 0.5566 | Regression_energy: 2.2633 | Regression_eta: 0.4664 | Regression_pt: 2.4945 | Regression_sin_phi: 0.9308 | LR: 9.67e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0857 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3041 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3174.6 | uss_MB: 2676.3 | child_rss_MB: 3218.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:22,568] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:38,827] INFO : Step 6200/50000 rank0 | Train Loss: 11.3747 | Train Loss Unweighted: 3.8074 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.3903 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1770 | Regression_eta: 0.0133 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9985 | Classification_binary: 3.3903 | Regression_cos_phi: 0.5511 | Regression_energy: 2.3830 | Regression_eta: 0.5582 | Regression_pt: 2.5675 | Regression_sin_phi: 0.9261 | LR: 9.66e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1046s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0029 | max_backward: 0.0342 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2860 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3175.6 | uss_MB: 2678.1 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:39,043] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:55,486] INFO : Step 6300/50000 rank0 | Train Loss: 10.3960 | Train Loss Unweighted: 3.7173 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.3363 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1626 | Regression_eta: 0.0091 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8930 | Classification_binary: 3.3363 | Regression_cos_phi: 0.4672 | Regression_energy: 2.1891 | Regression_eta: 0.3813 | Regression_pt: 2.4205 | Regression_sin_phi: 0.7086 | LR: 9.65e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.3216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1119 | max_loss: 0.0031 | max_backward: 0.0344 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3895 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3175.6 | uss_MB: 2678.3 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:55,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:11,928] INFO : Step 6400/50000 rank0 | Train Loss: 11.2701 | Train Loss Unweighted: 3.7425 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.3212 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1801 | Regression_eta: 0.0133 | Regression_pt: 0.1783 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9750 | Classification_binary: 3.3212 | Regression_cos_phi: 0.5876 | Regression_energy: 2.4240 | Regression_eta: 0.5591 | Regression_pt: 2.6080 | Regression_sin_phi: 0.7951 | LR: 9.64e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1983s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0029 | max_backward: 0.0337 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 3175.6 | uss_MB: 2678.6 | child_rss_MB: 3217.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:12,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:28,676] INFO : Step 6500/50000 rank0 | Train Loss: 11.3810 | Train Loss Unweighted: 3.7540 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.3339 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1786 | Regression_eta: 0.0134 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9364 | Classification_binary: 3.3339 | Regression_cos_phi: 0.6222 | Regression_energy: 2.4039 | Regression_eta: 0.5623 | Regression_pt: 2.5943 | Regression_sin_phi: 0.9281 | LR: 9.63e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3175.6 | uss_MB: 2678.7 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:35:28,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:45,191] INFO : Step 6600/50000 rank0 | Train Loss: 10.7826 | Train Loss Unweighted: 3.7040 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.3184 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1621 | Regression_eta: 0.0113 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9654 | Classification_binary: 3.3184 | Regression_cos_phi: 0.5792 | Regression_energy: 2.1826 | Regression_eta: 0.4721 | Regression_pt: 2.3658 | Regression_sin_phi: 0.8992 | LR: 9.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2929s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0897 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.6 | uss_MB: 2678.9 | child_rss_MB: 3212.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:35:45,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:01,966] INFO : Step 6700/50000 rank0 | Train Loss: 10.9594 | Train Loss Unweighted: 3.6751 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.2768 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1681 | Regression_eta: 0.0131 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9101 | Classification_binary: 3.2768 | Regression_cos_phi: 0.6282 | Regression_energy: 2.2629 | Regression_eta: 0.5509 | Regression_pt: 2.4506 | Regression_sin_phi: 0.8799 | LR: 9.61e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.0337 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3176.6 | uss_MB: 2679.2 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:02,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:18,634] INFO : Step 6800/50000 rank0 | Train Loss: 10.6312 | Train Loss Unweighted: 3.6811 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.2920 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1672 | Regression_eta: 0.0120 | Regression_pt: 0.1642 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8820 | Classification_binary: 3.2920 | Regression_cos_phi: 0.5566 | Regression_energy: 2.2515 | Regression_eta: 0.5019 | Regression_pt: 2.4011 | Regression_sin_phi: 0.7460 | LR: 9.60e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0425 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1071 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3176.9 | uss_MB: 2678.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:36:18,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:34,723] INFO : Step 6900/50000 rank0 | Train Loss: 10.4078 | Train Loss Unweighted: 3.5532 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.1589 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1720 | Regression_eta: 0.0084 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8811 | Classification_binary: 3.1589 | Regression_cos_phi: 0.5015 | Regression_energy: 2.3162 | Regression_eta: 0.3541 | Regression_pt: 2.4806 | Regression_sin_phi: 0.7153 | LR: 9.58e-04 | DataLoad Time: 0.1567s | Model Forward Time: 1.3090s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1199 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.3822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.9 | uss_MB: 2679.0 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:34,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:51,259] INFO : Step 7000/50000 rank0 | Train Loss: 11.0857 | Train Loss Unweighted: 3.6867 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.2770 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1745 | Regression_eta: 0.0124 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9206 | Classification_binary: 3.2770 | Regression_cos_phi: 0.5937 | Regression_energy: 2.3498 | Regression_eta: 0.5189 | Regression_pt: 2.5382 | Regression_sin_phi: 0.8875 | LR: 9.57e-04 | DataLoad Time: 0.0868s | Model Forward Time: 1.2075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.1285 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.9 | uss_MB: 2679.2 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:51,475] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:51,493] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 11:36:51,573] INFO : Running validation on rank0 +rank0 [2026-10-28 11:36:51,574] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:36:51,574] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:36:52,733] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 11:36:52,877] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:36:53,008] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:36:53,081] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:36:53,148] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:36:53,217] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:36:53,285] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:36:53,353] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:36:53,463] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:36:53,585] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:36:53,706] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:36:53,777] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:36:53,845] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:36:53,913] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:36:53,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:36:54,050] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:36:54,154] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:36:54,161] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:36:54,164] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:36:54,167] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:36:54,168] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:36:54,170] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:36:54,173] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:36:54,175] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:36:54,177] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:36:54,179] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:36:54,182] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:36:54,183] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:36:54,185] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:36:54,186] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:36:54,189] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:36:54,191] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:36:59,974] INFO : VALIDATION | Step=7000/50000 | Train Loss=11.0857 | Valid Loss=10.8608 | Train Loss Unweighted=3.6867 | Valid Loss Unweighted=3.6578 | Stale=0 | ETA=150.1m +rank0 [2026-10-28 11:36:59,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:00,046] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:37:00,049] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:00,198] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:01,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:02,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:02,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:03,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:03,642] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:37:03,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:03,701] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:37:03,704] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:03,759] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:05,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:05,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:06,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:06,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:06,948] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:06,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:07,005] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:37:07,008] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:37:07,088] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:08,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:08,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:09,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:10,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:10,089] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:10,097] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:10,346] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:10,794] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:12,228] INFO : Plotted jets +rank0 [2026-10-28 11:37:13,308] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:13,340] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:13,563] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:13,941] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:15,711] INFO : Plotted jets +rank0 [2026-10-28 11:37:16,818] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:16,851] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 11:37:17,052] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 11:37:17,478] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:18,863] INFO : Plotted jets +rank0 [2026-10-28 11:37:19,953] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:36,286] INFO : Step 7100/50000 rank0 | Train Loss: 10.7771 | Train Loss Unweighted: 3.6153 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2144 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1711 | Regression_eta: 0.0107 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9263 | Classification_binary: 3.2144 | Regression_cos_phi: 0.6110 | Regression_energy: 2.3034 | Regression_eta: 0.4493 | Regression_pt: 2.4975 | Regression_sin_phi: 0.7752 | LR: 9.56e-04 | DataLoad Time: 0.1419s | Model Forward Time: 1.1482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0034 | max_backward: 0.0342 | max_optimizer_step: 0.1092 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3397.6 | uss_MB: 2900.2 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2652.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:37:36,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 40.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:52,604] INFO : Step 7200/50000 rank0 | Train Loss: 10.6453 | Train Loss Unweighted: 3.7113 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.3190 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1688 | Regression_eta: 0.0098 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8429 | Classification_binary: 3.3190 | Regression_cos_phi: 0.4806 | Regression_energy: 2.2728 | Regression_eta: 0.4112 | Regression_pt: 2.4767 | Regression_sin_phi: 0.8420 | LR: 9.55e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2032 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3254.2 | uss_MB: 2757.7 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:37:52,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:08,717] INFO : Step 7300/50000 rank0 | Train Loss: 10.9155 | Train Loss Unweighted: 3.6839 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.2727 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1766 | Regression_eta: 0.0101 | Regression_pt: 0.1765 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9756 | Classification_binary: 3.2727 | Regression_cos_phi: 0.5119 | Regression_energy: 2.3773 | Regression_eta: 0.4252 | Regression_pt: 2.5813 | Regression_sin_phi: 0.7714 | LR: 9.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3254.2 | uss_MB: 2757.9 | child_rss_MB: 3219.0 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:38:08,942] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:24,928] INFO : Step 7400/50000 rank0 | Train Loss: 11.1385 | Train Loss Unweighted: 3.5861 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.1780 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1735 | Regression_eta: 0.0117 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9001 | Classification_binary: 3.1780 | Regression_cos_phi: 0.6423 | Regression_energy: 2.3354 | Regression_eta: 0.4896 | Regression_pt: 2.4980 | Regression_sin_phi: 1.0950 | LR: 9.52e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0756 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.2956 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3254.2 | uss_MB: 2758.1 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:38:25,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:41,004] INFO : Step 7500/50000 rank0 | Train Loss: 11.0638 | Train Loss Unweighted: 3.6412 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2261 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1806 | Regression_eta: 0.0118 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9014 | Classification_binary: 3.2261 | Regression_cos_phi: 0.6198 | Regression_energy: 2.4314 | Regression_eta: 0.4961 | Regression_pt: 2.5463 | Regression_sin_phi: 0.8426 | LR: 9.51e-04 | DataLoad Time: 0.0332s | Model Forward Time: 1.2194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3254.2 | uss_MB: 2758.7 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:41,230] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:56,790] INFO : Step 7600/50000 rank0 | Train Loss: 10.2642 | Train Loss Unweighted: 3.5164 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.1436 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1590 | Regression_eta: 0.0113 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8040 | Classification_binary: 3.1436 | Regression_cos_phi: 0.5561 | Regression_energy: 2.1399 | Regression_eta: 0.4721 | Regression_pt: 2.3129 | Regression_sin_phi: 0.8356 | LR: 9.50e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0564 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3254.2 | uss_MB: 2758.8 | child_rss_MB: 3217.3 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:57,000] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:12,858] INFO : Step 7700/50000 rank0 | Train Loss: 11.1161 | Train Loss Unweighted: 3.7196 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.2971 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1827 | Regression_eta: 0.0112 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8649 | Classification_binary: 3.2971 | Regression_cos_phi: 0.5456 | Regression_energy: 2.4597 | Regression_eta: 0.4713 | Regression_pt: 2.6761 | Regression_sin_phi: 0.8015 | LR: 9.48e-04 | DataLoad Time: 0.0949s | Model Forward Time: 1.1715s | Phase Times: max_pre_step_barrier: 0.0647 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3254.2 | uss_MB: 2758.9 | child_rss_MB: 3214.9 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:39:13,070] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:28,887] INFO : Step 7800/50000 rank0 | Train Loss: 11.2819 | Train Loss Unweighted: 3.6456 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.2264 | Regression_cos_phi: 0.0135 | Regression_energy: 0.1764 | Regression_eta: 0.0145 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9625 | Classification_binary: 3.2264 | Regression_cos_phi: 0.7370 | Regression_energy: 2.3743 | Regression_eta: 0.6065 | Regression_pt: 2.5799 | Regression_sin_phi: 0.7953 | LR: 9.47e-04 | DataLoad Time: 0.0973s | Model Forward Time: 1.2379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0452 | max_loss: 0.0033 | max_backward: 0.0350 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3761 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3253.8 | uss_MB: 2758.6 | child_rss_MB: 3213.9 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:39:29,098] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:45,071] INFO : Step 7900/50000 rank0 | Train Loss: 11.0550 | Train Loss Unweighted: 3.5752 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1742 | Regression_eta: 0.0117 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8664 | Classification_binary: 3.1659 | Regression_cos_phi: 0.6852 | Regression_energy: 2.3445 | Regression_eta: 0.4918 | Regression_pt: 2.5315 | Regression_sin_phi: 0.9698 | LR: 9.46e-04 | DataLoad Time: 0.1254s | Model Forward Time: 1.2358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0294 | max_forward: 0.0151 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3253.8 | uss_MB: 2759.1 | child_rss_MB: 3216.2 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:39:45,283] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:40:01,184] INFO : Step 8000/50000 rank0 | Train Loss: 10.6105 | Train Loss Unweighted: 3.5674 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.1674 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1719 | Regression_eta: 0.0104 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9768 | Classification_binary: 3.1674 | Regression_cos_phi: 0.4904 | Regression_energy: 2.3146 | Regression_eta: 0.4339 | Regression_pt: 2.4958 | Regression_sin_phi: 0.7316 | LR: 9.44e-04 | DataLoad Time: 0.0853s | Model Forward Time: 1.2833s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0771 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3253.8 | uss_MB: 2759.2 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:40:01,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:01,463] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 11:40:01,544] INFO : Running validation on rank0 +rank0 [2026-10-28 11:40:01,544] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:40:01,544] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:40:02,618] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 11:40:02,761] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:40:02,892] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:40:02,966] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:40:03,034] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:40:03,102] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:40:03,171] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:40:03,240] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:40:03,350] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:40:03,481] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:40:03,611] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:40:03,682] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:40:03,751] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:40:03,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:40:03,888] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:40:03,957] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:40:04,065] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:40:04,071] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:40:04,074] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:40:04,075] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:40:04,078] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:40:04,080] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:40:04,081] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:40:04,084] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:40:04,086] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:40:04,087] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:40:04,090] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:40:04,091] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:40:04,094] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:40:04,096] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:40:04,097] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:40:04,099] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:40:09,887] INFO : VALIDATION | Step=8000/50000 | Train Loss=10.6105 | Valid Loss=10.7036 | Train Loss Unweighted=3.5674 | Valid Loss Unweighted=3.5624 | Stale=0 | ETA=144.9m +rank0 [2026-10-28 11:40:09,896] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:09,957] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:40:09,961] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:10,108] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:11,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:12,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:12,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:13,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:13,368] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:13,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:13,427] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:40:13,431] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:13,527] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:14,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:15,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:15,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:16,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:16,613] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:16,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:16,671] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:40:16,674] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:40:16,755] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:18,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:18,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:19,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:19,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:19,748] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:19,755] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:20,006] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:20,456] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:21,888] INFO : Plotted jets +rank0 [2026-10-28 11:40:22,970] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:23,001] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:23,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:24,042] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:25,429] INFO : Plotted jets +rank0 [2026-10-28 11:40:26,547] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:26,579] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 11:40:26,780] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 11:40:27,205] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:28,601] INFO : Plotted jets +rank0 [2026-10-28 11:40:29,719] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:45,696] INFO : Step 8100/50000 rank0 | Train Loss: 10.6482 | Train Loss Unweighted: 3.5311 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.1403 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1644 | Regression_eta: 0.0114 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8861 | Classification_binary: 3.1403 | Regression_cos_phi: 0.5756 | Regression_energy: 2.2126 | Regression_eta: 0.4775 | Regression_pt: 2.4397 | Regression_sin_phi: 0.9165 | LR: 9.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2029s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0944 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3463.2 | uss_MB: 2970.4 | child_rss_MB: 3218.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:40:45,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:01,459] INFO : Step 8200/50000 rank0 | Train Loss: 10.4078 | Train Loss Unweighted: 3.6404 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.2632 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1597 | Regression_eta: 0.0109 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8726 | Classification_binary: 3.2632 | Regression_cos_phi: 0.4951 | Regression_energy: 2.1495 | Regression_eta: 0.4580 | Regression_pt: 2.3642 | Regression_sin_phi: 0.8052 | LR: 9.42e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0440 | max_forward: 0.0155 | max_loss: 0.0030 | max_backward: 0.0338 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3325.1 | uss_MB: 2832.1 | child_rss_MB: 3220.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:41:01,670] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:17,227] INFO : Step 8300/50000 rank0 | Train Loss: 10.2977 | Train Loss Unweighted: 3.5546 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.1660 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1671 | Regression_eta: 0.0102 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8283 | Classification_binary: 3.1660 | Regression_cos_phi: 0.5127 | Regression_energy: 2.2502 | Regression_eta: 0.4260 | Regression_pt: 2.4780 | Regression_sin_phi: 0.6365 | LR: 9.40e-04 | DataLoad Time: 0.0549s | Model Forward Time: 1.1751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0032 | max_backward: 0.0784 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3326.1 | uss_MB: 2832.3 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:41:17,439] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:32,984] INFO : Step 8400/50000 rank0 | Train Loss: 10.7513 | Train Loss Unweighted: 3.5155 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1055 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1777 | Regression_eta: 0.0114 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8422 | Classification_binary: 3.1055 | Regression_cos_phi: 0.5666 | Regression_energy: 2.3924 | Regression_eta: 0.4779 | Regression_pt: 2.5673 | Regression_sin_phi: 0.7994 | LR: 9.39e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2580s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0648 | max_forward: 0.0155 | max_loss: 0.0030 | max_backward: 0.0340 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3726 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3326.1 | uss_MB: 2832.5 | child_rss_MB: 3214.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:33,199] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:48,696] INFO : Step 8500/50000 rank0 | Train Loss: 10.3485 | Train Loss Unweighted: 3.4752 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.0994 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1576 | Regression_eta: 0.0115 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8820 | Classification_binary: 3.0994 | Regression_cos_phi: 0.5769 | Regression_energy: 2.1220 | Regression_eta: 0.4806 | Regression_pt: 2.3288 | Regression_sin_phi: 0.8587 | LR: 9.37e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0325 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3326.1 | uss_MB: 2832.8 | child_rss_MB: 3217.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:48,923] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:04,190] INFO : Step 8600/50000 rank0 | Train Loss: 10.0229 | Train Loss Unweighted: 3.4751 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.1060 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1568 | Regression_eta: 0.0099 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8706 | Classification_binary: 3.1060 | Regression_cos_phi: 0.5192 | Regression_energy: 2.1110 | Regression_eta: 0.4147 | Regression_pt: 2.3191 | Regression_sin_phi: 0.6822 | LR: 9.36e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1790s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.1017 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3326.1 | uss_MB: 2832.9 | child_rss_MB: 3216.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:04,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:19,999] INFO : Step 8700/50000 rank0 | Train Loss: 9.9227 | Train Loss Unweighted: 3.3353 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.9779 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1502 | Regression_eta: 0.0111 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8722 | Classification_binary: 2.9779 | Regression_cos_phi: 0.5139 | Regression_energy: 2.0222 | Regression_eta: 0.4662 | Regression_pt: 2.1917 | Regression_sin_phi: 0.8787 | LR: 9.34e-04 | DataLoad Time: 0.1212s | Model Forward Time: 1.1749s | Phase Times: max_pre_step_barrier: 0.0551 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3327.1 | uss_MB: 2833.3 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:42:20,214] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:35,704] INFO : Step 8800/50000 rank0 | Train Loss: 11.5046 | Train Loss Unweighted: 3.6919 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.2655 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1785 | Regression_eta: 0.0149 | Regression_pt: 0.1801 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9619 | Classification_binary: 3.2655 | Regression_cos_phi: 0.7142 | Regression_energy: 2.4035 | Regression_eta: 0.6245 | Regression_pt: 2.6339 | Regression_sin_phi: 0.9011 | LR: 9.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2339s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3325.7 | uss_MB: 2832.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:35,935] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:51,401] INFO : Step 8900/50000 rank0 | Train Loss: 10.7739 | Train Loss Unweighted: 3.4304 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.0253 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1729 | Regression_eta: 0.0111 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9002 | Classification_binary: 3.0253 | Regression_cos_phi: 0.5458 | Regression_energy: 2.3275 | Regression_eta: 0.4669 | Regression_pt: 2.5159 | Regression_sin_phi: 0.9923 | LR: 9.31e-04 | DataLoad Time: 0.0926s | Model Forward Time: 1.3070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0786 | max_backward: 0.0319 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 3326.7 | uss_MB: 2832.0 | child_rss_MB: 3213.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:42:51,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:06,970] INFO : Step 9000/50000 rank0 | Train Loss: 9.8280 | Train Loss Unweighted: 3.4087 | Train Loss Components: Classification: 0.0270 | Classification_binary: 3.0559 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1480 | Regression_eta: 0.0109 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9151 | Classification_binary: 3.0559 | Regression_cos_phi: 0.5171 | Regression_energy: 1.9930 | Regression_eta: 0.4577 | Regression_pt: 2.1686 | Regression_sin_phi: 0.7206 | LR: 9.30e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0214 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3326.7 | uss_MB: 2833.4 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:43:07,184] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:07,258] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 11:43:07,340] INFO : Running validation on rank0 +rank0 [2026-10-28 11:43:07,340] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:43:07,341] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:43:08,434] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 11:43:08,577] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:43:08,708] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:43:08,781] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:43:08,849] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:43:08,916] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:43:08,985] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:43:09,055] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:43:09,162] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:43:09,285] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:43:09,405] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:43:09,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:43:09,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:43:09,614] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:43:09,682] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:43:09,750] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:43:09,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:43:09,871] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:43:09,873] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:43:09,874] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:43:09,877] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:43:09,879] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:43:09,881] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:43:09,883] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:43:09,886] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:43:09,887] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:43:09,890] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:43:09,892] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:43:09,893] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:43:09,896] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:43:09,896] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:43:09,899] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:43:15,951] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.8280 | Valid Loss=10.4704 | Train Loss Unweighted=3.4087 | Valid Loss Unweighted=3.5190 | Stale=0 | ETA=139.9m +rank0 [2026-10-28 11:43:15,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:16,026] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:43:16,029] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:16,179] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:17,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:18,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:18,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:19,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:19,392] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:19,395] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:19,451] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:43:19,454] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:19,534] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:20,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:21,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:21,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:22,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:22,710] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:22,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:22,767] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:43:22,770] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:43:22,827] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:24,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:24,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:25,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:25,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:26,003] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:26,011] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:26,262] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:26,718] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:28,156] INFO : Plotted jets +rank0 [2026-10-28 11:43:29,239] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:29,272] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:29,501] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:29,877] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:31,702] INFO : Plotted jets +rank0 [2026-10-28 11:43:32,811] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:32,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 11:43:33,044] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 11:43:33,459] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:34,853] INFO : Plotted jets +rank0 [2026-10-28 11:43:35,941] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:51,678] INFO : Step 9100/50000 rank0 | Train Loss: 10.2210 | Train Loss Unweighted: 3.4537 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.0720 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1647 | Regression_eta: 0.0105 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7939 | Classification_binary: 3.0720 | Regression_cos_phi: 0.5389 | Regression_energy: 2.2175 | Regression_eta: 0.4383 | Regression_pt: 2.3909 | Regression_sin_phi: 0.7695 | LR: 9.28e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1523s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3081 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 3533.0 | uss_MB: 3040.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:43:51,894] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:07,599] INFO : Step 9200/50000 rank0 | Train Loss: 10.2051 | Train Loss Unweighted: 3.4388 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.0578 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1628 | Regression_eta: 0.0101 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8480 | Classification_binary: 3.0578 | Regression_cos_phi: 0.5230 | Regression_energy: 2.1916 | Regression_eta: 0.4228 | Regression_pt: 2.3962 | Regression_sin_phi: 0.7658 | LR: 9.27e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2612s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0431 | max_loss: 0.0030 | max_backward: 0.0347 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3402.4 | uss_MB: 2908.4 | child_rss_MB: 3210.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:44:07,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:23,154] INFO : Step 9300/50000 rank0 | Train Loss: 9.7815 | Train Loss Unweighted: 3.4304 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.0844 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1460 | Regression_eta: 0.0100 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8392 | Classification_binary: 3.0844 | Regression_cos_phi: 0.5702 | Regression_energy: 1.9661 | Regression_eta: 0.4194 | Regression_pt: 2.1195 | Regression_sin_phi: 0.7828 | LR: 9.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3402.4 | uss_MB: 2909.9 | child_rss_MB: 3214.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:44:23,370] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:38,575] INFO : Step 9400/50000 rank0 | Train Loss: 10.1318 | Train Loss Unweighted: 3.4839 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1170 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1558 | Regression_eta: 0.0112 | Regression_pt: 0.1540 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8917 | Classification_binary: 3.1170 | Regression_cos_phi: 0.5528 | Regression_energy: 2.0969 | Regression_eta: 0.4691 | Regression_pt: 2.2522 | Regression_sin_phi: 0.7521 | LR: 9.24e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1370s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3402.4 | uss_MB: 2910.1 | child_rss_MB: 3217.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:44:38,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:54,180] INFO : Step 9500/50000 rank0 | Train Loss: 10.3209 | Train Loss Unweighted: 3.4558 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.0716 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1654 | Regression_eta: 0.0113 | Regression_pt: 0.1625 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8669 | Classification_binary: 3.0716 | Regression_cos_phi: 0.5013 | Regression_energy: 2.2264 | Regression_eta: 0.4755 | Regression_pt: 2.3773 | Regression_sin_phi: 0.8019 | LR: 9.22e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1390 | max_optimizer_step: 0.1077 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3402.4 | uss_MB: 2910.2 | child_rss_MB: 3217.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:44:54,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:09,613] INFO : Step 9600/50000 rank0 | Train Loss: 10.6374 | Train Loss Unweighted: 3.4569 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.0632 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1671 | Regression_eta: 0.0120 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9034 | Classification_binary: 3.0632 | Regression_cos_phi: 0.6219 | Regression_energy: 2.2493 | Regression_eta: 0.5010 | Regression_pt: 2.4199 | Regression_sin_phi: 0.8787 | LR: 9.21e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0852 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3402.4 | uss_MB: 2910.3 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:09,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:25,324] INFO : Step 9700/50000 rank0 | Train Loss: 10.1423 | Train Loss Unweighted: 3.4407 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.0752 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1511 | Regression_eta: 0.0120 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8524 | Classification_binary: 3.0752 | Regression_cos_phi: 0.6542 | Regression_energy: 2.0336 | Regression_eta: 0.5017 | Regression_pt: 2.2804 | Regression_sin_phi: 0.7448 | LR: 9.19e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2420s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0403 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3402.4 | uss_MB: 2910.4 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:25,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:40,776] INFO : Step 9800/50000 rank0 | Train Loss: 10.7870 | Train Loss Unweighted: 3.5506 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.1493 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1713 | Regression_eta: 0.0116 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8614 | Classification_binary: 3.1493 | Regression_cos_phi: 0.5457 | Regression_energy: 2.3058 | Regression_eta: 0.4876 | Regression_pt: 2.5032 | Regression_sin_phi: 0.9340 | LR: 9.17e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1283s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.2904 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3400.1 | uss_MB: 2908.9 | child_rss_MB: 3214.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:40,993] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:56,065] INFO : Step 9900/50000 rank0 | Train Loss: 10.3658 | Train Loss Unweighted: 3.4258 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.0458 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1601 | Regression_eta: 0.0108 | Regression_pt: 0.1599 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9304 | Classification_binary: 3.0458 | Regression_cos_phi: 0.6235 | Regression_energy: 2.1556 | Regression_eta: 0.4533 | Regression_pt: 2.3386 | Regression_sin_phi: 0.8187 | LR: 9.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2939s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0797 | max_forward: 0.0154 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3402.1 | uss_MB: 2910.4 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:56,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:11,489] INFO : Step 10000/50000 rank0 | Train Loss: 12.1191 | Train Loss Unweighted: 3.6639 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.1969 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1976 | Regression_eta: 0.0164 | Regression_pt: 0.1997 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9570 | Classification_binary: 3.1969 | Regression_cos_phi: 0.6321 | Regression_energy: 2.6608 | Regression_eta: 0.6867 | Regression_pt: 2.9212 | Regression_sin_phi: 1.0643 | LR: 9.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0984 | max_forward: 0.0165 | max_loss: 0.0032 | max_backward: 0.0385 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3775 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3403.1 | uss_MB: 2912.9 | child_rss_MB: 3214.6 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:46:11,703] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:11,742] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 11:46:11,824] INFO : Running validation on rank0 +rank0 [2026-10-28 11:46:11,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:46:11,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:46:12,898] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 11:46:13,041] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:46:13,172] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:46:13,246] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:46:13,315] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:46:13,382] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:46:13,451] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:46:13,520] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:46:13,628] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:46:13,751] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:46:13,871] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:46:13,943] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:46:14,013] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:46:14,082] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:46:14,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:46:14,219] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:46:14,327] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:46:14,334] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:46:14,336] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:46:14,339] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:46:14,340] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:46:14,343] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:46:14,345] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:46:14,346] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:46:14,349] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:46:14,351] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:46:14,352] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:46:14,355] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:46:14,356] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:46:14,358] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:46:14,361] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:46:14,362] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:46:20,152] INFO : VALIDATION | Step=10000/50000 | Train Loss=12.1191 | Valid Loss=10.3040 | Train Loss Unweighted=3.6639 | Valid Loss Unweighted=3.4715 | Stale=0 | ETA=135.1m +rank0 [2026-10-28 11:46:20,164] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:20,227] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:46:20,231] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:20,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:21,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:22,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:22,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:23,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:23,620] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:23,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:23,680] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:46:23,683] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:23,768] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:25,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:25,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:26,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:26,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:27,055] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:27,058] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:27,111] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:46:27,115] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:46:27,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:28,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:29,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:29,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:30,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:30,167] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:30,174] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:30,423] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:30,875] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:32,318] INFO : Plotted jets +rank0 [2026-10-28 11:46:33,408] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:33,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:33,666] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:34,072] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:35,922] INFO : Plotted jets +rank0 [2026-10-28 11:46:37,038] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:37,201] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 11:46:37,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 11:46:37,825] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:39,226] INFO : Plotted jets +rank0 [2026-10-28 11:46:40,320] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:55,629] INFO : Step 10100/50000 rank0 | Train Loss: 10.8258 | Train Loss Unweighted: 3.5885 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.1945 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1663 | Regression_eta: 0.0126 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8990 | Classification_binary: 3.1945 | Regression_cos_phi: 0.6024 | Regression_energy: 2.2392 | Regression_eta: 0.5295 | Regression_pt: 2.4227 | Regression_sin_phi: 0.9384 | LR: 9.12e-04 | DataLoad Time: 0.0229s | Model Forward Time: 1.2621s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.1341 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.2898 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3609.3 | uss_MB: 3116.8 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:46:55,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:10,755] INFO : Step 10200/50000 rank0 | Train Loss: 10.2172 | Train Loss Unweighted: 3.4195 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.0523 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1542 | Regression_eta: 0.0120 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8723 | Classification_binary: 3.0523 | Regression_cos_phi: 0.6242 | Regression_energy: 2.0755 | Regression_eta: 0.5037 | Regression_pt: 2.2400 | Regression_sin_phi: 0.8493 | LR: 9.11e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0516 | max_forward: 0.0138 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3478.1 | uss_MB: 2985.7 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:47:10,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:26,041] INFO : Step 10300/50000 rank0 | Train Loss: 9.6742 | Train Loss Unweighted: 3.4153 | Train Loss Components: Classification: 0.0243 | Classification_binary: 3.0612 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1510 | Regression_eta: 0.0086 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 3.0612 | Regression_cos_phi: 0.4229 | Regression_energy: 2.0325 | Regression_eta: 0.3623 | Regression_pt: 2.2393 | Regression_sin_phi: 0.7298 | LR: 9.09e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3242s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1244 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3478.1 | uss_MB: 2987.1 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:26,255] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:41,367] INFO : Step 10400/50000 rank0 | Train Loss: 10.5939 | Train Loss Unweighted: 3.5083 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1250 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1641 | Regression_eta: 0.0139 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8422 | Classification_binary: 3.1250 | Regression_cos_phi: 0.7059 | Regression_energy: 2.2085 | Regression_eta: 0.5829 | Regression_pt: 2.2959 | Regression_sin_phi: 0.8335 | LR: 9.07e-04 | DataLoad Time: 0.0092s | Model Forward Time: 1.3180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0788 | max_backward: 0.0332 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3478.1 | uss_MB: 2986.4 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:41,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:56,232] INFO : Step 10500/50000 rank0 | Train Loss: 10.1197 | Train Loss Unweighted: 3.4256 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0461 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1629 | Regression_eta: 0.0097 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8907 | Classification_binary: 3.0461 | Regression_cos_phi: 0.5410 | Regression_energy: 2.1931 | Regression_eta: 0.4048 | Regression_pt: 2.3747 | Regression_sin_phi: 0.6693 | LR: 9.06e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1059 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3478.1 | uss_MB: 2987.8 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:56,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:11,295] INFO : Step 10600/50000 rank0 | Train Loss: 9.5424 | Train Loss Unweighted: 3.3392 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.9886 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1505 | Regression_eta: 0.0099 | Regression_pt: 0.1500 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7794 | Classification_binary: 2.9886 | Regression_cos_phi: 0.4746 | Regression_energy: 2.0263 | Regression_eta: 0.4162 | Regression_pt: 2.1944 | Regression_sin_phi: 0.6629 | LR: 9.04e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2139s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0683 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.2997 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3478.1 | uss_MB: 2987.9 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:48:11,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:26,580] INFO : Step 10700/50000 rank0 | Train Loss: 9.3631 | Train Loss Unweighted: 3.2919 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9586 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1420 | Regression_eta: 0.0120 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7967 | Classification_binary: 2.9586 | Regression_cos_phi: 0.4644 | Regression_energy: 1.9119 | Regression_eta: 0.5020 | Regression_pt: 2.0242 | Regression_sin_phi: 0.7054 | LR: 9.02e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2359s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.0345 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.2897 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3478.1 | uss_MB: 2988.0 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:48:26,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:41,976] INFO : Step 10800/50000 rank0 | Train Loss: 9.8531 | Train Loss Unweighted: 3.3195 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9526 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1569 | Regression_eta: 0.0086 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8329 | Classification_binary: 2.9526 | Regression_cos_phi: 0.4783 | Regression_energy: 2.1123 | Regression_eta: 0.3586 | Regression_pt: 2.3082 | Regression_sin_phi: 0.8101 | LR: 9.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0029 | max_backward: 0.0331 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3478.1 | uss_MB: 2988.0 | child_rss_MB: 3215.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:42,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:57,412] INFO : Step 10900/50000 rank0 | Train Loss: 10.1457 | Train Loss Unweighted: 3.3871 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.0010 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1664 | Regression_eta: 0.0102 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9144 | Classification_binary: 3.0010 | Regression_cos_phi: 0.4578 | Regression_energy: 2.2404 | Regression_eta: 0.4294 | Regression_pt: 2.4196 | Regression_sin_phi: 0.6830 | LR: 8.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2633s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.3063 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3477.0 | uss_MB: 2987.4 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:57,648] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:12,972] INFO : Step 11000/50000 rank0 | Train Loss: 10.1478 | Train Loss Unweighted: 3.3844 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.0058 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1620 | Regression_eta: 0.0106 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8561 | Classification_binary: 3.0058 | Regression_cos_phi: 0.5701 | Regression_energy: 2.1805 | Regression_eta: 0.4425 | Regression_pt: 2.3554 | Regression_sin_phi: 0.7373 | LR: 8.97e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2578s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 3477.0 | uss_MB: 2987.8 | child_rss_MB: 3214.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:49:13,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:13,223] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 11:49:13,303] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:13,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:13,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:14,496] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 11:49:14,640] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:14,771] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:14,844] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:14,912] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:14,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:15,048] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:15,116] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:15,228] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:15,358] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:15,488] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:15,558] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:15,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:15,693] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:15,762] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:15,830] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:15,938] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:15,945] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:15,947] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:15,950] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:15,951] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:15,954] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:15,957] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:15,960] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:15,961] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:15,964] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:15,967] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:15,970] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:15,971] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:15,973] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:15,974] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:15,977] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:21,902] INFO : VALIDATION | Step=11000/50000 | Train Loss=10.1478 | Valid Loss=10.1514 | Train Loss Unweighted=3.3844 | Valid Loss Unweighted=3.4104 | Stale=0 | ETA=130.5m +rank0 [2026-10-28 11:49:21,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:21,974] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:21,977] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:22,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:23,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:24,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:24,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:25,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:25,416] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:25,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:25,475] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:25,478] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:25,640] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:26,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:27,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:28,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:28,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:28,758] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:28,762] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:28,816] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:28,819] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:49:28,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:30,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:30,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:31,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:31,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:31,836] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:31,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:32,096] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:32,549] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:33,988] INFO : Plotted jets +rank0 [2026-10-28 11:49:35,071] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:35,103] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:35,328] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:35,705] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:37,549] INFO : Plotted jets +rank0 [2026-10-28 11:49:38,664] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:38,697] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:38,898] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:39,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:40,709] INFO : Plotted jets +rank0 [2026-10-28 11:49:41,805] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:57,070] INFO : Step 11100/50000 rank0 | Train Loss: 9.9509 | Train Loss Unweighted: 3.3484 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9855 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1554 | Regression_eta: 0.0107 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8041 | Classification_binary: 2.9855 | Regression_cos_phi: 0.6330 | Regression_energy: 2.0927 | Regression_eta: 0.4473 | Regression_pt: 2.2195 | Regression_sin_phi: 0.7687 | LR: 8.95e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2637s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0166 | max_loss: 0.0032 | max_backward: 0.1387 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3683.8 | uss_MB: 3196.4 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:49:57,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:12,186] INFO : Step 11200/50000 rank0 | Train Loss: 9.7615 | Train Loss Unweighted: 3.3100 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1483 | Regression_eta: 0.0117 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8991 | Classification_binary: 2.9575 | Regression_cos_phi: 0.4938 | Regression_energy: 1.9964 | Regression_eta: 0.4907 | Regression_pt: 2.1527 | Regression_sin_phi: 0.7712 | LR: 8.93e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0943 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.3623 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3683.8 | uss_MB: 3196.5 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:50:12,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:26,912] INFO : Step 11300/50000 rank0 | Train Loss: 9.8666 | Train Loss Unweighted: 3.3011 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9421 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1533 | Regression_eta: 0.0129 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7992 | Classification_binary: 2.9421 | Regression_cos_phi: 0.5864 | Regression_energy: 2.0637 | Regression_eta: 0.5403 | Regression_pt: 2.1794 | Regression_sin_phi: 0.7556 | LR: 8.91e-04 | DataLoad Time: 0.1135s | Model Forward Time: 1.1713s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1087 | max_tensorboard_logging: 0.3009 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 3548.8 | uss_MB: 3063.0 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:50:27,124] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:41,706] INFO : Step 11400/50000 rank0 | Train Loss: 9.7117 | Train Loss Unweighted: 3.3258 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9925 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1367 | Regression_eta: 0.0139 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8040 | Classification_binary: 2.9925 | Regression_cos_phi: 0.6689 | Regression_energy: 1.8404 | Regression_eta: 0.5832 | Regression_pt: 1.9909 | Regression_sin_phi: 0.8317 | LR: 8.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1181s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1091 | max_tensorboard_logging: 0.2847 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3549.8 | uss_MB: 3063.1 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:50:41,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:56,686] INFO : Step 11500/50000 rank0 | Train Loss: 10.2846 | Train Loss Unweighted: 3.3150 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.9364 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1591 | Regression_eta: 0.0161 | Regression_pt: 0.1579 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8368 | Classification_binary: 2.9364 | Regression_cos_phi: 0.5886 | Regression_energy: 2.1423 | Regression_eta: 0.6741 | Regression_pt: 2.3096 | Regression_sin_phi: 0.7969 | LR: 8.88e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0616 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3550.8 | uss_MB: 3063.3 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:50:56,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:11,731] INFO : Step 11600/50000 rank0 | Train Loss: 10.0478 | Train Loss Unweighted: 3.2852 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9071 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1616 | Regression_eta: 0.0112 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8658 | Classification_binary: 2.9071 | Regression_cos_phi: 0.5044 | Regression_energy: 2.1749 | Regression_eta: 0.4687 | Regression_pt: 2.3527 | Regression_sin_phi: 0.7742 | LR: 8.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0530 | max_loss: 0.0031 | max_backward: 0.0325 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3551.8 | uss_MB: 3063.4 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:51:11,948] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:26,575] INFO : Step 11700/50000 rank0 | Train Loss: 10.6136 | Train Loss Unweighted: 3.4505 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0729 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1599 | Regression_eta: 0.0145 | Regression_pt: 0.1521 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8882 | Classification_binary: 3.0729 | Regression_cos_phi: 0.6670 | Regression_energy: 2.1524 | Regression_eta: 0.6077 | Regression_pt: 2.2242 | Regression_sin_phi: 1.0011 | LR: 8.84e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1359s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3551.8 | uss_MB: 3063.6 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:51:26,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:41,174] INFO : Step 11800/50000 rank0 | Train Loss: 11.0674 | Train Loss Unweighted: 3.4144 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9898 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1837 | Regression_eta: 0.0145 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8057 | Classification_binary: 2.9898 | Regression_cos_phi: 0.6092 | Regression_energy: 2.4730 | Regression_eta: 0.6092 | Regression_pt: 2.6231 | Regression_sin_phi: 0.9574 | LR: 8.82e-04 | DataLoad Time: 0.0644s | Model Forward Time: 1.2657s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0545 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3491 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3551.8 | uss_MB: 3063.6 | child_rss_MB: 3209.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:51:41,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:56,465] INFO : Step 11900/50000 rank0 | Train Loss: 9.2926 | Train Loss Unweighted: 3.2014 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.8618 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1440 | Regression_eta: 0.0075 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8465 | Classification_binary: 2.8618 | Regression_cos_phi: 0.5379 | Regression_energy: 1.9386 | Regression_eta: 0.3161 | Regression_pt: 2.1163 | Regression_sin_phi: 0.6754 | LR: 8.80e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1003 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3550.9 | uss_MB: 3063.1 | child_rss_MB: 3209.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:51:56,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:11,214] INFO : Step 12000/50000 rank0 | Train Loss: 9.3280 | Train Loss Unweighted: 3.2430 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.8970 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1491 | Regression_eta: 0.0097 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8079 | Classification_binary: 2.8970 | Regression_cos_phi: 0.4599 | Regression_energy: 2.0078 | Regression_eta: 0.4056 | Regression_pt: 2.1564 | Regression_sin_phi: 0.5933 | LR: 8.78e-04 | DataLoad Time: 0.1076s | Model Forward Time: 1.3308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0977 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3550.9 | uss_MB: 3063.2 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:52:11,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:11,541] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 11:52:11,624] INFO : Running validation on rank0 +rank0 [2026-10-28 11:52:11,624] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:52:11,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:52:12,716] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 11:52:12,859] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:52:12,991] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:52:13,067] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:52:13,138] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:52:13,206] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:52:13,274] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:52:13,344] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:52:13,455] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:52:13,587] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:52:13,718] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:52:13,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:52:13,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:52:13,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:52:13,998] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:52:14,068] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:52:14,183] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:52:14,190] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:52:14,191] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:52:14,194] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:52:14,197] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:52:14,199] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:52:14,201] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:52:14,203] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:52:14,204] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:52:14,207] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:52:14,210] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:52:14,212] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:52:14,215] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:52:14,217] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:52:14,219] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:52:14,220] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:52:20,305] INFO : VALIDATION | Step=12000/50000 | Train Loss=9.3280 | Valid Loss=9.9684 | Train Loss Unweighted=3.2430 | Valid Loss Unweighted=3.3511 | Stale=0 | ETA=126.0m +rank0 [2026-10-28 11:52:20,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:20,382] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:52:20,385] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:20,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:22,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:22,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:23,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:23,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:23,981] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:52:23,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:24,039] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:52:24,043] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:24,128] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:25,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:26,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:26,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:27,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:27,345] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:27,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:27,402] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:52:27,405] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:52:27,492] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:28,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:29,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:29,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:30,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:30,458] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:30,467] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:30,717] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:31,169] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:32,609] INFO : Plotted jets +rank0 [2026-10-28 11:52:33,692] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:33,725] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:33,949] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:34,327] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:36,165] INFO : Plotted jets +rank0 [2026-10-28 11:52:37,294] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:37,327] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 11:52:37,529] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 11:52:37,950] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:39,351] INFO : Plotted jets +rank0 [2026-10-28 11:52:40,453] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:55,408] INFO : Step 12100/50000 rank0 | Train Loss: 10.0726 | Train Loss Unweighted: 3.3186 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9450 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1590 | Regression_eta: 0.0131 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.9450 | Regression_cos_phi: 0.5372 | Regression_energy: 2.1408 | Regression_eta: 0.5487 | Regression_pt: 2.2918 | Regression_sin_phi: 0.7418 | LR: 8.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2512s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0343 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3459 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3763.5 | uss_MB: 3279.0 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:52:55,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:10,087] INFO : Step 12200/50000 rank0 | Train Loss: 9.7603 | Train Loss Unweighted: 3.3390 | Train Loss Components: Classification: 0.0260 | Classification_binary: 2.9841 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1500 | Regression_eta: 0.0110 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8837 | Classification_binary: 2.9841 | Regression_cos_phi: 0.4914 | Regression_energy: 2.0199 | Regression_eta: 0.4593 | Regression_pt: 2.1876 | Regression_sin_phi: 0.7342 | LR: 8.74e-04 | DataLoad Time: 0.1000s | Model Forward Time: 1.2326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0227 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3406 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3763.5 | uss_MB: 3279.1 | child_rss_MB: 3211.8 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:53:10,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:24,733] INFO : Step 12300/50000 rank0 | Train Loss: 9.8808 | Train Loss Unweighted: 3.3626 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9892 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1614 | Regression_eta: 0.0096 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7926 | Classification_binary: 2.9892 | Regression_cos_phi: 0.4996 | Regression_energy: 2.1726 | Regression_eta: 0.4029 | Regression_pt: 2.3618 | Regression_sin_phi: 0.6620 | LR: 8.72e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2966s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0877 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3624.4 | uss_MB: 3140.0 | child_rss_MB: 3209.9 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:24,947] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:39,186] INFO : Step 12400/50000 rank0 | Train Loss: 9.9525 | Train Loss Unweighted: 3.3426 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9756 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1557 | Regression_eta: 0.0150 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7377 | Classification_binary: 2.9756 | Regression_cos_phi: 0.5284 | Regression_energy: 2.0967 | Regression_eta: 0.6285 | Regression_pt: 2.2796 | Regression_sin_phi: 0.7060 | LR: 8.70e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2258s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0032 | max_backward: 0.1389 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3625.4 | uss_MB: 3140.3 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:39,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:53,616] INFO : Step 12500/50000 rank0 | Train Loss: 10.6162 | Train Loss Unweighted: 3.3865 | Train Loss Components: Classification: 0.0253 | Classification_binary: 2.9762 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1777 | Regression_eta: 0.0114 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.9762 | Regression_cos_phi: 0.5689 | Regression_energy: 2.3926 | Regression_eta: 0.4779 | Regression_pt: 2.5684 | Regression_sin_phi: 0.7720 | LR: 8.68e-04 | DataLoad Time: 0.0939s | Model Forward Time: 1.1895s | Phase Times: max_pre_step_barrier: 0.0653 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1102 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3625.4 | uss_MB: 3140.6 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:53,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:08,267] INFO : Step 12600/50000 rank0 | Train Loss: 10.0286 | Train Loss Unweighted: 3.2811 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.9085 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1597 | Regression_eta: 0.0139 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7704 | Classification_binary: 2.9085 | Regression_cos_phi: 0.4935 | Regression_energy: 2.1501 | Regression_eta: 0.5809 | Regression_pt: 2.2935 | Regression_sin_phi: 0.8317 | LR: 8.66e-04 | DataLoad Time: 0.1165s | Model Forward Time: 1.3459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1178 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3625.4 | uss_MB: 3140.8 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:08,478] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:22,649] INFO : Step 12700/50000 rank0 | Train Loss: 9.7353 | Train Loss Unweighted: 3.2943 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.9402 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1513 | Regression_eta: 0.0142 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7772 | Classification_binary: 2.9402 | Regression_cos_phi: 0.4766 | Regression_energy: 2.0374 | Regression_eta: 0.5958 | Regression_pt: 2.1567 | Regression_sin_phi: 0.7515 | LR: 8.64e-04 | DataLoad Time: 0.0567s | Model Forward Time: 1.1275s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0542 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2685 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3625.4 | uss_MB: 3140.9 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:22,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:37,358] INFO : Step 12800/50000 rank0 | Train Loss: 9.5525 | Train Loss Unweighted: 3.3974 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.0452 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1521 | Regression_eta: 0.0097 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 3.0452 | Regression_cos_phi: 0.4103 | Regression_energy: 2.0476 | Regression_eta: 0.4083 | Regression_pt: 2.1962 | Regression_sin_phi: 0.5863 | LR: 8.62e-04 | DataLoad Time: 0.0946s | Model Forward Time: 1.1456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0557 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3625.4 | uss_MB: 3140.9 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:54:37,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:51,753] INFO : Step 12900/50000 rank0 | Train Loss: 9.7605 | Train Loss Unweighted: 3.3315 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.9757 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1511 | Regression_eta: 0.0102 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8169 | Classification_binary: 2.9757 | Regression_cos_phi: 0.5302 | Regression_energy: 2.0340 | Regression_eta: 0.4256 | Regression_pt: 2.2079 | Regression_sin_phi: 0.7702 | LR: 8.60e-04 | DataLoad Time: 0.1048s | Model Forward Time: 1.1621s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3623.9 | uss_MB: 3140.3 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:54:51,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:06,361] INFO : Step 13000/50000 rank0 | Train Loss: 10.4666 | Train Loss Unweighted: 3.4348 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.0425 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1681 | Regression_eta: 0.0111 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8399 | Classification_binary: 3.0425 | Regression_cos_phi: 0.5639 | Regression_energy: 2.2626 | Regression_eta: 0.4651 | Regression_pt: 2.4472 | Regression_sin_phi: 0.8455 | LR: 8.58e-04 | DataLoad Time: 0.0927s | Model Forward Time: 1.2725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0729 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3623.9 | uss_MB: 3140.5 | child_rss_MB: 3207.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:06,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:06,591] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 11:55:06,778] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:06,779] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:06,779] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:07,855] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 11:55:07,999] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:08,131] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:08,205] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:08,276] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:08,344] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:08,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:08,483] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:08,594] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:08,716] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:08,839] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:08,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:08,982] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:09,050] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:09,119] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:09,190] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:09,299] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:09,306] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:09,309] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:09,312] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:09,312] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:09,315] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:09,318] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:09,319] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:09,322] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:09,323] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:09,325] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:09,328] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:09,331] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:09,332] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:09,335] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:09,338] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:15,353] INFO : VALIDATION | Step=13000/50000 | Train Loss=10.4666 | Valid Loss=9.8645 | Train Loss Unweighted=3.4348 | Valid Loss Unweighted=3.2896 | Stale=0 | ETA=121.5m +rank0 [2026-10-28 11:55:15,363] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:15,424] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:15,428] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:15,579] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:17,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:17,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:18,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:19,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:19,117] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:55:19,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:19,176] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:55:19,179] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:19,257] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:20,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:21,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:21,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:22,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:22,482] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:22,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:22,538] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:55:22,540] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:55:22,632] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:24,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:24,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:25,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:25,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:25,689] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:25,696] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:25,946] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:26,397] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:27,834] INFO : Plotted jets +rank0 [2026-10-28 11:55:28,918] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:28,949] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:29,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:29,549] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:31,353] INFO : Plotted jets +rank0 [2026-10-28 11:55:32,467] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:32,499] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 11:55:32,699] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 11:55:33,114] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:34,505] INFO : Plotted jets +rank0 [2026-10-28 11:55:35,593] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:49,999] INFO : Step 13100/50000 rank0 | Train Loss: 9.9021 | Train Loss Unweighted: 3.3099 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9571 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1481 | Regression_eta: 0.0102 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7999 | Classification_binary: 2.9571 | Regression_cos_phi: 0.5824 | Regression_energy: 1.9936 | Regression_eta: 0.4293 | Regression_pt: 2.1616 | Regression_sin_phi: 0.9780 | LR: 8.56e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0032 | max_backward: 0.0325 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.2712 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3814.4 | uss_MB: 3332.7 | child_rss_MB: 3205.8 | allocated_MB: 207.0 | reserved_MB: 2630.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:55:50,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:04,489] INFO : Step 13200/50000 rank0 | Train Loss: 9.4704 | Train Loss Unweighted: 3.2823 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9415 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1459 | Regression_eta: 0.0109 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7440 | Classification_binary: 2.9415 | Regression_cos_phi: 0.4974 | Regression_energy: 1.9648 | Regression_eta: 0.4579 | Regression_pt: 2.0933 | Regression_sin_phi: 0.7714 | LR: 8.54e-04 | DataLoad Time: 0.0987s | Model Forward Time: 1.3084s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0767 | max_backward: 0.0337 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3730 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3814.4 | uss_MB: 3331.5 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:56:04,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:18,717] INFO : Step 13300/50000 rank0 | Train Loss: 10.2088 | Train Loss Unweighted: 3.3526 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.9630 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1679 | Regression_eta: 0.0100 | Regression_pt: 0.1677 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8618 | Classification_binary: 2.9630 | Regression_cos_phi: 0.5021 | Regression_energy: 2.2602 | Regression_eta: 0.4202 | Regression_pt: 2.4519 | Regression_sin_phi: 0.7497 | LR: 8.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0615 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3694.6 | uss_MB: 3211.4 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:56:18,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:33,044] INFO : Step 13400/50000 rank0 | Train Loss: 9.8936 | Train Loss Unweighted: 3.3379 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.9810 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1510 | Regression_eta: 0.0114 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7942 | Classification_binary: 2.9810 | Regression_cos_phi: 0.6452 | Regression_energy: 2.0328 | Regression_eta: 0.4778 | Regression_pt: 2.1852 | Regression_sin_phi: 0.7775 | LR: 8.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2317s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.1153 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3694.6 | uss_MB: 3212.4 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:56:33,258] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:47,006] INFO : Step 13500/50000 rank0 | Train Loss: 10.1506 | Train Loss Unweighted: 3.3300 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9547 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1593 | Regression_eta: 0.0142 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.9547 | Regression_cos_phi: 0.5353 | Regression_energy: 2.1441 | Regression_eta: 0.5945 | Regression_pt: 2.3128 | Regression_sin_phi: 0.8032 | LR: 8.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0164 | max_loss: 0.0032 | max_backward: 0.0721 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2746 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3695.6 | uss_MB: 3215.3 | child_rss_MB: 3210.5 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:56:47,220] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:01,604] INFO : Step 13600/50000 rank0 | Train Loss: 10.3087 | Train Loss Unweighted: 3.3338 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.9396 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1709 | Regression_eta: 0.0117 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.9396 | Regression_cos_phi: 0.5880 | Regression_energy: 2.3003 | Regression_eta: 0.4887 | Regression_pt: 2.4650 | Regression_sin_phi: 0.7496 | LR: 8.45e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2389s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3695.6 | uss_MB: 3213.4 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:01,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:15,689] INFO : Step 13700/50000 rank0 | Train Loss: 10.0445 | Train Loss Unweighted: 3.3482 | Train Loss Components: Classification: 0.0262 | Classification_binary: 2.9761 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1580 | Regression_eta: 0.0116 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.9761 | Regression_cos_phi: 0.5853 | Regression_energy: 2.1275 | Regression_eta: 0.4880 | Regression_pt: 2.2931 | Regression_sin_phi: 0.6851 | LR: 8.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2143s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0179 | max_loss: 0.0033 | max_backward: 0.1040 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3695.6 | uss_MB: 3215.7 | child_rss_MB: 3211.3 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:57:15,904] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:29,967] INFO : Step 13800/50000 rank0 | Train Loss: 9.8036 | Train Loss Unweighted: 3.2180 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.8653 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1484 | Regression_eta: 0.0113 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8381 | Classification_binary: 2.8653 | Regression_cos_phi: 0.5862 | Regression_energy: 1.9976 | Regression_eta: 0.4718 | Regression_pt: 2.1371 | Regression_sin_phi: 0.9077 | LR: 8.41e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3250s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1174 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3695.6 | uss_MB: 3213.7 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:30,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:44,292] INFO : Step 13900/50000 rank0 | Train Loss: 9.5682 | Train Loss Unweighted: 3.3080 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.9676 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1446 | Regression_eta: 0.0114 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8158 | Classification_binary: 2.9676 | Regression_cos_phi: 0.4844 | Regression_energy: 1.9467 | Regression_eta: 0.4760 | Regression_pt: 2.0648 | Regression_sin_phi: 0.8128 | LR: 8.39e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3694.1 | uss_MB: 3211.7 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:44,522] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:58,373] INFO : Step 14000/50000 rank0 | Train Loss: 9.9458 | Train Loss Unweighted: 3.2759 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9074 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1601 | Regression_eta: 0.0135 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.9074 | Regression_cos_phi: 0.5622 | Regression_energy: 2.1548 | Regression_eta: 0.5651 | Regression_pt: 2.2421 | Regression_sin_phi: 0.7881 | LR: 8.37e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1722s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0678 | max_optimizer_step: 0.1125 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3695.1 | uss_MB: 3212.7 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:57:58,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:58,607] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 11:57:58,687] INFO : Running validation on rank0 +rank0 [2026-10-28 11:57:58,688] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:57:58,688] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:57:59,871] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 11:58:00,016] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:58:00,148] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:58:00,222] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:58:00,290] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:58:00,358] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:58:00,428] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:58:00,499] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:58:00,607] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:58:00,730] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:58:00,853] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:58:00,925] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:58:00,993] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:58:01,063] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:58:01,134] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:58:01,204] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:58:01,311] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:58:01,317] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:58:01,321] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:58:01,324] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:58:01,325] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:58:01,328] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:58:01,331] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:58:01,332] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:58:01,335] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:58:01,337] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:58:01,340] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:58:01,341] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:58:01,343] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:58:01,345] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:58:01,347] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:58:01,350] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:58:07,199] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.9458 | Valid Loss=9.9386 | Train Loss Unweighted=3.2759 | Valid Loss Unweighted=3.2938 | Stale=1 | ETA=117.2m +rank0 [2026-10-28 11:58:07,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:07,270] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:58:07,273] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:07,337] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:08,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:09,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:10,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:10,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:10,749] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:58:10,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:10,809] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:58:10,811] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:10,969] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:12,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:12,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:13,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:14,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:14,230] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:14,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:14,287] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:58:14,303] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:58:14,459] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:15,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:16,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:16,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:17,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:17,529] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:17,537] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:17,862] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:18,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:19,760] INFO : Plotted jets +rank0 [2026-10-28 11:58:20,853] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:20,888] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:21,114] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:21,494] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:23,307] INFO : Plotted jets +rank0 [2026-10-28 11:58:24,431] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:24,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 11:58:24,663] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 11:58:25,083] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:26,490] INFO : Plotted jets +rank0 [2026-10-28 11:58:27,604] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:41,581] INFO : Step 14100/50000 rank0 | Train Loss: 10.0267 | Train Loss Unweighted: 3.2078 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8400 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1546 | Regression_eta: 0.0118 | Regression_pt: 0.1543 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.8400 | Regression_cos_phi: 0.5396 | Regression_energy: 2.0811 | Regression_eta: 0.4950 | Regression_pt: 2.2569 | Regression_sin_phi: 0.9638 | LR: 8.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2002s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0833 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.2730 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3903.4 | uss_MB: 3420.7 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:58:41,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:55,861] INFO : Step 14200/50000 rank0 | Train Loss: 10.3101 | Train Loss Unweighted: 3.3419 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.9599 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1653 | Regression_eta: 0.0141 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.9599 | Regression_cos_phi: 0.5479 | Regression_energy: 2.2250 | Regression_eta: 0.5898 | Regression_pt: 2.3156 | Regression_sin_phi: 0.8962 | LR: 8.32e-04 | DataLoad Time: 0.1271s | Model Forward Time: 1.2848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0740 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.3673 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3903.4 | uss_MB: 3420.8 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:58:56,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:09,912] INFO : Step 14300/50000 rank0 | Train Loss: 10.8830 | Train Loss Unweighted: 3.3773 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9602 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1742 | Regression_eta: 0.0168 | Regression_pt: 0.1803 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8642 | Classification_binary: 2.9602 | Regression_cos_phi: 0.5206 | Regression_energy: 2.3449 | Regression_eta: 0.7049 | Regression_pt: 2.6367 | Regression_sin_phi: 0.8515 | LR: 8.30e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1051 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1091 | max_tensorboard_logging: 0.3620 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3769.1 | uss_MB: 3286.2 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:59:10,127] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:24,198] INFO : Step 14400/50000 rank0 | Train Loss: 9.9240 | Train Loss Unweighted: 3.2902 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.9117 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1644 | Regression_eta: 0.0113 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8286 | Classification_binary: 2.9117 | Regression_cos_phi: 0.4625 | Regression_energy: 2.2138 | Regression_eta: 0.4756 | Regression_pt: 2.3593 | Regression_sin_phi: 0.6723 | LR: 8.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.1406 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2752 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3769.1 | uss_MB: 3287.3 | child_rss_MB: 3213.0 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:24,410] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:38,495] INFO : Step 14500/50000 rank0 | Train Loss: 9.5049 | Train Loss Unweighted: 3.1723 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.8152 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1517 | Regression_eta: 0.0093 | Regression_pt: 0.1541 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8140 | Classification_binary: 2.8152 | Regression_cos_phi: 0.5138 | Regression_energy: 2.0427 | Regression_eta: 0.3890 | Regression_pt: 2.2532 | Regression_sin_phi: 0.6769 | LR: 8.26e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0738 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1090 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3769.1 | uss_MB: 3287.4 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:59:38,707] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:52,395] INFO : Step 14600/50000 rank0 | Train Loss: 9.9713 | Train Loss Unweighted: 3.2409 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8667 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1596 | Regression_eta: 0.0110 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8127 | Classification_binary: 2.8667 | Regression_cos_phi: 0.5104 | Regression_energy: 2.1480 | Regression_eta: 0.4624 | Regression_pt: 2.3361 | Regression_sin_phi: 0.8350 | LR: 8.24e-04 | DataLoad Time: 0.0851s | Model Forward Time: 1.3219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0826 | max_backward: 0.0328 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3769.1 | uss_MB: 3286.8 | child_rss_MB: 3212.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:59:52,607] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:06,144] INFO : Step 14700/50000 rank0 | Train Loss: 9.7108 | Train Loss Unweighted: 3.2602 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8926 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1589 | Regression_eta: 0.0123 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7975 | Classification_binary: 2.8926 | Regression_cos_phi: 0.4664 | Regression_energy: 2.1397 | Regression_eta: 0.5149 | Regression_pt: 2.2919 | Regression_sin_phi: 0.6078 | LR: 8.21e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1850s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3769.1 | uss_MB: 3287.9 | child_rss_MB: 3213.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:06,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:19,933] INFO : Step 14800/50000 rank0 | Train Loss: 10.4445 | Train Loss Unweighted: 3.3912 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.0009 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1689 | Regression_eta: 0.0132 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8549 | Classification_binary: 3.0009 | Regression_cos_phi: 0.5854 | Regression_energy: 2.2740 | Regression_eta: 0.5543 | Regression_pt: 2.3690 | Regression_sin_phi: 0.8059 | LR: 8.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1389s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3769.1 | uss_MB: 3288.1 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:00:20,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:34,128] INFO : Step 14900/50000 rank0 | Train Loss: 10.1577 | Train Loss Unweighted: 3.3607 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.9874 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1596 | Regression_eta: 0.0115 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8101 | Classification_binary: 2.9874 | Regression_cos_phi: 0.5326 | Regression_energy: 2.1488 | Regression_eta: 0.4829 | Regression_pt: 2.2996 | Regression_sin_phi: 0.8963 | LR: 8.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3768.0 | uss_MB: 3286.5 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:00:34,358] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:47,891] INFO : Step 15000/50000 rank0 | Train Loss: 8.5458 | Train Loss Unweighted: 3.0886 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.7865 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1271 | Regression_eta: 0.0103 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8249 | Classification_binary: 2.7865 | Regression_cos_phi: 0.3969 | Regression_energy: 1.7109 | Regression_eta: 0.4336 | Regression_pt: 1.8447 | Regression_sin_phi: 0.5482 | LR: 8.15e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.1365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1102 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3769.0 | uss_MB: 3287.6 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:48,104] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:48,151] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 12:00:48,232] INFO : Running validation on rank0 +rank0 [2026-10-28 12:00:48,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:00:48,233] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:00:49,325] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 12:00:49,470] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:00:49,602] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:00:49,677] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:00:49,746] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:00:49,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:00:49,884] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:00:49,955] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:00:50,065] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:00:50,199] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:00:50,331] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:00:50,403] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:00:50,473] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:00:50,542] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:00:50,612] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:00:50,682] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:00:50,793] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:00:50,800] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:00:50,803] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:00:50,806] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:00:50,807] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:00:50,810] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:00:50,812] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:00:50,815] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:00:50,816] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:00:50,818] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:00:50,819] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:00:50,822] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:00:50,932] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:00:50,935] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:00:50,938] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:00:50,941] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:00:56,781] INFO : VALIDATION | Step=15000/50000 | Train Loss=8.5458 | Valid Loss=9.7435 | Train Loss Unweighted=3.0886 | Valid Loss Unweighted=3.2473 | Stale=0 | ETA=112.9m +rank0 [2026-10-28 12:00:56,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:00:56,852] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:00:56,855] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:00:57,013] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:00:58,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:00:59,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:00:59,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:00,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:00,440] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:01:00,443] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:00,499] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:01:00,503] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:00,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:01,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:02,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:03,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:03,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:03,653] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:03,655] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:03,709] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:01:03,712] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:01:03,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:05,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:05,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:06,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:06,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:06,937] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:06,944] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:01:07,193] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:01:07,646] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:09,087] INFO : Plotted jets +rank0 [2026-10-28 12:01:10,170] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:10,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:10,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:10,809] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:12,647] INFO : Plotted jets +rank0 [2026-10-28 12:01:13,763] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:13,795] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 12:01:13,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 12:01:14,415] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:15,815] INFO : Plotted jets +rank0 [2026-10-28 12:01:16,906] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:31,091] INFO : Step 15100/50000 rank0 | Train Loss: 9.5909 | Train Loss Unweighted: 3.2524 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.9163 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1391 | Regression_eta: 0.0098 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.9163 | Regression_cos_phi: 0.5772 | Regression_energy: 1.8724 | Regression_eta: 0.4116 | Regression_pt: 2.0552 | Regression_sin_phi: 0.9319 | LR: 8.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3683s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1221 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3981.8 | uss_MB: 3501.1 | child_rss_MB: 3213.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:31,305] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:45,033] INFO : Step 15200/50000 rank0 | Train Loss: 9.4751 | Train Loss Unweighted: 3.2279 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.8864 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1447 | Regression_eta: 0.0109 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8325 | Classification_binary: 2.8864 | Regression_cos_phi: 0.5447 | Regression_energy: 1.9476 | Regression_eta: 0.4562 | Regression_pt: 2.0813 | Regression_sin_phi: 0.7265 | LR: 8.10e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2817s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0279 | max_backward: 0.0336 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3981.8 | uss_MB: 3499.9 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:01:45,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:58,752] INFO : Step 15300/50000 rank0 | Train Loss: 10.3137 | Train Loss Unweighted: 3.2825 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.8944 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1653 | Regression_eta: 0.0124 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8737 | Classification_binary: 2.8944 | Regression_cos_phi: 0.5425 | Regression_energy: 2.2249 | Regression_eta: 0.5202 | Regression_pt: 2.3960 | Regression_sin_phi: 0.8619 | LR: 8.08e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0039 | max_backward: 0.0325 | max_optimizer_step: 0.1550 | max_tensorboard_logging: 0.2703 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3981.8 | uss_MB: 3501.2 | child_rss_MB: 3208.1 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:58,983] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:12,483] INFO : Step 15400/50000 rank0 | Train Loss: 9.5617 | Train Loss Unweighted: 3.2062 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8592 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1461 | Regression_eta: 0.0112 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.8592 | Regression_cos_phi: 0.4559 | Regression_energy: 1.9663 | Regression_eta: 0.4684 | Regression_pt: 2.1519 | Regression_sin_phi: 0.8700 | LR: 8.05e-04 | DataLoad Time: 0.0476s | Model Forward Time: 1.2243s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0333 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.0375 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3844.9 | uss_MB: 3366.3 | child_rss_MB: 3208.0 | allocated_MB: 210.7 | reserved_MB: 2748.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:02:12,698] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:26,169] INFO : Step 15500/50000 rank0 | Train Loss: 9.2195 | Train Loss Unweighted: 3.1396 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.7910 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1507 | Regression_eta: 0.0099 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8364 | Classification_binary: 2.7910 | Regression_cos_phi: 0.4372 | Regression_energy: 2.0294 | Regression_eta: 0.4146 | Regression_pt: 2.1711 | Regression_sin_phi: 0.5398 | LR: 8.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2409s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.1410 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3844.9 | uss_MB: 3364.7 | child_rss_MB: 3208.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:26,383] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:39,934] INFO : Step 15600/50000 rank0 | Train Loss: 9.8572 | Train Loss Unweighted: 3.3209 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9505 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1611 | Regression_eta: 0.0098 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7359 | Classification_binary: 2.9505 | Regression_cos_phi: 0.4797 | Regression_energy: 2.1686 | Regression_eta: 0.4114 | Regression_pt: 2.3272 | Regression_sin_phi: 0.7838 | LR: 8.01e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2791s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0532 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3844.9 | uss_MB: 3365.0 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:02:40,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:53,553] INFO : Step 15700/50000 rank0 | Train Loss: 9.8435 | Train Loss Unweighted: 3.3038 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.9473 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1515 | Regression_eta: 0.0123 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8204 | Classification_binary: 2.9473 | Regression_cos_phi: 0.5148 | Regression_energy: 2.0394 | Regression_eta: 0.5169 | Regression_pt: 2.1727 | Regression_sin_phi: 0.8321 | LR: 7.98e-04 | DataLoad Time: 0.0216s | Model Forward Time: 1.2396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.1372 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3844.9 | uss_MB: 3365.0 | child_rss_MB: 3207.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:53,766] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:07,231] INFO : Step 15800/50000 rank0 | Train Loss: 9.3396 | Train Loss Unweighted: 3.0977 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.7469 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1495 | Regression_eta: 0.0083 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8360 | Classification_binary: 2.7469 | Regression_cos_phi: 0.4908 | Regression_energy: 2.0130 | Regression_eta: 0.3499 | Regression_pt: 2.2000 | Regression_sin_phi: 0.7030 | LR: 7.96e-04 | DataLoad Time: 0.0491s | Model Forward Time: 1.3089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0785 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3844.9 | uss_MB: 3365.1 | child_rss_MB: 3210.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:07,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:21,288] INFO : Step 15900/50000 rank0 | Train Loss: 10.4359 | Train Loss Unweighted: 3.3478 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.9619 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1626 | Regression_eta: 0.0126 | Regression_pt: 0.1634 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8246 | Classification_binary: 2.9619 | Regression_cos_phi: 0.5949 | Regression_energy: 2.1896 | Regression_eta: 0.5271 | Regression_pt: 2.3903 | Regression_sin_phi: 0.9474 | LR: 7.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2724s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0616 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.3400 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3844.9 | uss_MB: 3365.1 | child_rss_MB: 3210.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:21,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:34,864] INFO : Step 16000/50000 rank0 | Train Loss: 9.8188 | Train Loss Unweighted: 3.1922 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8361 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1482 | Regression_eta: 0.0130 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7992 | Classification_binary: 2.8361 | Regression_cos_phi: 0.6768 | Regression_energy: 1.9949 | Regression_eta: 0.5435 | Regression_pt: 2.1790 | Regression_sin_phi: 0.7895 | LR: 7.91e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2315s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0324 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3844.5 | uss_MB: 3364.4 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:35,090] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:35,107] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 12:03:35,188] INFO : Running validation on rank0 +rank0 [2026-10-28 12:03:35,188] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:03:35,188] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:03:36,363] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 12:03:36,509] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:03:36,642] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:03:36,717] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:03:36,785] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:03:36,853] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:03:36,923] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:03:36,993] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:03:37,106] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:03:37,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:03:37,355] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:03:37,428] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:03:37,497] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:03:37,566] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:03:37,636] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:03:37,705] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:03:37,807] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:03:37,813] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:03:37,816] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:03:37,819] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:03:37,820] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:03:37,823] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:03:37,826] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:03:37,829] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:03:37,832] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:03:37,833] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:03:37,836] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:03:37,838] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:03:37,840] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:03:37,843] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:03:37,845] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:03:37,848] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:03:43,668] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.8188 | Valid Loss=9.5773 | Train Loss Unweighted=3.1922 | Valid Loss Unweighted=3.1936 | Stale=0 | ETA=108.7m +rank0 [2026-10-28 12:03:43,677] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:43,739] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:03:43,743] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:43,901] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:45,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:45,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:46,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:47,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:47,178] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:47,181] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:47,236] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:03:47,239] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:47,334] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:48,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:49,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:49,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:50,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:50,632] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:50,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:50,689] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:03:50,691] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:03:50,783] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:52,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:52,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:53,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:53,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:53,818] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:53,826] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:54,077] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:54,532] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:55,971] INFO : Plotted jets +rank0 [2026-10-28 12:03:57,058] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:57,091] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:57,316] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:57,695] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:59,511] INFO : Plotted jets +rank0 [2026-10-28 12:04:00,625] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:00,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 12:04:00,860] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 12:04:01,277] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:02,674] INFO : Plotted jets +rank0 [2026-10-28 12:04:03,773] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:17,442] INFO : Step 16100/50000 rank0 | Train Loss: 9.6267 | Train Loss Unweighted: 3.2122 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8726 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1441 | Regression_eta: 0.0126 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7871 | Classification_binary: 2.8726 | Regression_cos_phi: 0.5837 | Regression_energy: 1.9395 | Regression_eta: 0.5275 | Regression_pt: 2.0125 | Regression_sin_phi: 0.9037 | LR: 7.89e-04 | DataLoad Time: 0.0824s | Model Forward Time: 1.2839s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0808 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.3327 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:17,655] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:30,996] INFO : Step 16200/50000 rank0 | Train Loss: 10.2974 | Train Loss Unweighted: 3.2839 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8981 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1617 | Regression_eta: 0.0129 | Regression_pt: 0.1648 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8475 | Classification_binary: 2.8981 | Regression_cos_phi: 0.6017 | Regression_energy: 2.1765 | Regression_eta: 0.5411 | Regression_pt: 2.4106 | Regression_sin_phi: 0.8221 | LR: 7.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2193s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3209.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:31,226] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:44,709] INFO : Step 16300/50000 rank0 | Train Loss: 8.7498 | Train Loss Unweighted: 3.0414 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7248 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1342 | Regression_eta: 0.0103 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.7248 | Regression_cos_phi: 0.4333 | Regression_energy: 1.8071 | Regression_eta: 0.4301 | Regression_pt: 1.9544 | Regression_sin_phi: 0.6353 | LR: 7.84e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0588 | max_loss: 0.0031 | max_backward: 0.0324 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:04:44,923] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:58,296] INFO : Step 16400/50000 rank0 | Train Loss: 9.2254 | Train Loss Unweighted: 3.1327 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.7997 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1396 | Regression_eta: 0.0123 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8719 | Classification_binary: 2.7997 | Regression_cos_phi: 0.5175 | Regression_energy: 1.8789 | Regression_eta: 0.5172 | Regression_pt: 2.0196 | Regression_sin_phi: 0.6206 | LR: 7.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2245s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.2720 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:58,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:11,668] INFO : Step 16500/50000 rank0 | Train Loss: 9.6383 | Train Loss Unweighted: 3.2608 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9035 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1520 | Regression_eta: 0.0109 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7572 | Classification_binary: 2.9035 | Regression_cos_phi: 0.5213 | Regression_energy: 2.0467 | Regression_eta: 0.4570 | Regression_pt: 2.2459 | Regression_sin_phi: 0.7066 | LR: 7.79e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2564s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.1372 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:11,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:25,269] INFO : Step 16600/50000 rank0 | Train Loss: 9.0491 | Train Loss Unweighted: 3.1575 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8307 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1384 | Regression_eta: 0.0069 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8124 | Classification_binary: 2.8307 | Regression_cos_phi: 0.5092 | Regression_energy: 1.8633 | Regression_eta: 0.2907 | Regression_pt: 2.0368 | Regression_sin_phi: 0.7060 | LR: 7.77e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0334 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:25,483] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:38,959] INFO : Step 16700/50000 rank0 | Train Loss: 9.2943 | Train Loss Unweighted: 3.1391 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8098 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1372 | Regression_eta: 0.0109 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.8098 | Regression_cos_phi: 0.5442 | Regression_energy: 1.8465 | Regression_eta: 0.4557 | Regression_pt: 2.0053 | Regression_sin_phi: 0.8269 | LR: 7.74e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.1390 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3918.6 | uss_MB: 3438.3 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:39,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:52,457] INFO : Step 16800/50000 rank0 | Train Loss: 10.1016 | Train Loss Unweighted: 3.2139 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8307 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1632 | Regression_eta: 0.0113 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8491 | Classification_binary: 2.8307 | Regression_cos_phi: 0.5265 | Regression_energy: 2.1965 | Regression_eta: 0.4753 | Regression_pt: 2.3913 | Regression_sin_phi: 0.8322 | LR: 7.72e-04 | DataLoad Time: 0.0466s | Model Forward Time: 1.3442s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0921 | max_backward: 0.0339 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3918.6 | uss_MB: 3437.1 | child_rss_MB: 3204.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:52,670] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:05,837] INFO : Step 16900/50000 rank0 | Train Loss: 9.5274 | Train Loss Unweighted: 3.1959 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.8448 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1498 | Regression_eta: 0.0099 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7722 | Classification_binary: 2.8448 | Regression_cos_phi: 0.5667 | Regression_energy: 2.0165 | Regression_eta: 0.4133 | Regression_pt: 2.1786 | Regression_sin_phi: 0.7352 | LR: 7.69e-04 | DataLoad Time: 0.0725s | Model Forward Time: 1.2738s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0582 | max_loss: 0.0030 | max_backward: 0.0330 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.3729 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3918.6 | uss_MB: 3438.4 | child_rss_MB: 3207.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:06,048] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:19,113] INFO : Step 17000/50000 rank0 | Train Loss: 10.1798 | Train Loss Unweighted: 3.2488 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8548 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1678 | Regression_eta: 0.0122 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7508 | Classification_binary: 2.8548 | Regression_cos_phi: 0.5539 | Regression_energy: 2.2594 | Regression_eta: 0.5107 | Regression_pt: 2.5214 | Regression_sin_phi: 0.7286 | LR: 7.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3224s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0861 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3709 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3917.1 | uss_MB: 3438.3 | child_rss_MB: 3204.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:06:19,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:19,346] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 12:06:19,516] INFO : Running validation on rank0 +rank0 [2026-10-28 12:06:19,516] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:06:19,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:06:20,586] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 12:06:20,729] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:06:20,860] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:06:20,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:06:21,006] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:06:21,073] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:06:21,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:06:21,211] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:06:21,322] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:06:21,446] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:06:21,567] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:06:21,639] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:06:21,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:06:21,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:06:21,844] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:06:21,913] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:06:22,016] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:06:22,022] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:06:22,025] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:06:22,028] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:06:22,029] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:06:22,032] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:06:22,035] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:06:22,037] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:06:22,040] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:06:22,041] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:06:22,044] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:06:22,046] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:06:22,049] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:06:22,050] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:06:22,053] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:06:22,056] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:06:27,831] INFO : VALIDATION | Step=17000/50000 | Train Loss=10.1798 | Valid Loss=9.4903 | Train Loss Unweighted=3.2488 | Valid Loss Unweighted=3.1783 | Stale=0 | ETA=104.6m +rank0 [2026-10-28 12:06:27,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:27,902] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:06:27,906] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:28,065] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:29,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:30,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:30,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:31,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:31,423] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:06:31,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:31,483] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:06:31,487] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:31,576] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:33,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:33,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:34,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:34,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:34,753] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:06:34,756] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:34,811] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:06:34,814] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:06:34,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:36,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:36,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:37,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:37,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:38,030] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:06:38,038] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:38,287] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:38,738] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:40,175] INFO : Plotted jets +rank0 [2026-10-28 12:06:41,257] INFO : Plotted jet ratio +rank0 [2026-10-28 12:06:41,298] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:41,521] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:41,898] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:43,714] INFO : Plotted jets +rank0 [2026-10-28 12:06:44,822] INFO : Plotted jet ratio +rank0 [2026-10-28 12:06:44,855] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 12:06:45,055] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 12:06:45,470] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:46,862] INFO : Plotted jets +rank0 [2026-10-28 12:06:47,954] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:01,580] INFO : Step 17100/50000 rank0 | Train Loss: 9.4942 | Train Loss Unweighted: 3.3069 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9519 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1540 | Regression_eta: 0.0099 | Regression_pt: 0.1525 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.9519 | Regression_cos_phi: 0.4056 | Regression_energy: 2.0728 | Regression_eta: 0.4130 | Regression_pt: 2.2306 | Regression_sin_phi: 0.6302 | LR: 7.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.1432 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4112.9 | uss_MB: 3639.3 | child_rss_MB: 3202.8 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:07:01,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:14,842] INFO : Step 17200/50000 rank0 | Train Loss: 9.3799 | Train Loss Unweighted: 3.2388 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9048 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1404 | Regression_eta: 0.0124 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7911 | Classification_binary: 2.9048 | Regression_cos_phi: 0.5194 | Regression_energy: 1.8905 | Regression_eta: 0.5204 | Regression_pt: 2.0361 | Regression_sin_phi: 0.7176 | LR: 7.62e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0039 | max_backward: 0.1396 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4112.9 | uss_MB: 3637.3 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:07:15,057] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:28,203] INFO : Step 17300/50000 rank0 | Train Loss: 9.6138 | Train Loss Unweighted: 3.2544 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8936 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1546 | Regression_eta: 0.0119 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8044 | Classification_binary: 2.8936 | Regression_cos_phi: 0.4122 | Regression_energy: 2.0814 | Regression_eta: 0.4992 | Regression_pt: 2.2608 | Regression_sin_phi: 0.6621 | LR: 7.59e-04 | DataLoad Time: 0.0528s | Model Forward Time: 1.2566s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.1338 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2753 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4112.9 | uss_MB: 3637.4 | child_rss_MB: 3202.8 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:07:28,418] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:41,333] INFO : Step 17400/50000 rank0 | Train Loss: 9.5478 | Train Loss Unweighted: 3.2325 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8744 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1560 | Regression_eta: 0.0089 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7050 | Classification_binary: 2.8744 | Regression_cos_phi: 0.5162 | Regression_energy: 2.1005 | Regression_eta: 0.3744 | Regression_pt: 2.2473 | Regression_sin_phi: 0.7300 | LR: 7.57e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1938s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0030 | max_backward: 0.0324 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.2958 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3990.5 | uss_MB: 3514.0 | child_rss_MB: 3201.7 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:07:41,547] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:54,772] INFO : Step 17500/50000 rank0 | Train Loss: 8.8813 | Train Loss Unweighted: 3.0143 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6815 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1427 | Regression_eta: 0.0088 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7668 | Classification_binary: 2.6815 | Regression_cos_phi: 0.4575 | Regression_energy: 1.9214 | Regression_eta: 0.3684 | Regression_pt: 2.0870 | Regression_sin_phi: 0.5988 | LR: 7.54e-04 | DataLoad Time: 0.0263s | Model Forward Time: 1.1540s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0343 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3991.5 | uss_MB: 3514.6 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:07:55,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:08,167] INFO : Step 17600/50000 rank0 | Train Loss: 9.6255 | Train Loss Unweighted: 3.0583 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.7203 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1400 | Regression_eta: 0.0154 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7703 | Classification_binary: 2.7203 | Regression_cos_phi: 0.6761 | Regression_energy: 1.8846 | Regression_eta: 0.6474 | Regression_pt: 1.9807 | Regression_sin_phi: 0.9462 | LR: 7.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0888 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3991.5 | uss_MB: 3514.7 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:08:08,381] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:21,472] INFO : Step 17700/50000 rank0 | Train Loss: 9.1505 | Train Loss Unweighted: 3.1639 | Train Loss Components: Classification: 0.0251 | Classification_binary: 2.8279 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1425 | Regression_eta: 0.0086 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8532 | Classification_binary: 2.8279 | Regression_cos_phi: 0.4143 | Regression_energy: 1.9181 | Regression_eta: 0.3606 | Regression_pt: 2.1020 | Regression_sin_phi: 0.6744 | LR: 7.49e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2296s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0189 | max_loss: 0.0032 | max_backward: 0.0954 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3992.5 | uss_MB: 3517.2 | child_rss_MB: 3200.4 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:08:21,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:34,676] INFO : Step 17800/50000 rank0 | Train Loss: 8.9580 | Train Loss Unweighted: 2.9660 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6336 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1427 | Regression_eta: 0.0132 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7675 | Classification_binary: 2.6336 | Regression_cos_phi: 0.4313 | Regression_energy: 1.9214 | Regression_eta: 0.5524 | Regression_pt: 2.0167 | Regression_sin_phi: 0.6351 | LR: 7.47e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3992.5 | uss_MB: 3515.2 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:08:34,905] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:48,218] INFO : Step 17900/50000 rank0 | Train Loss: 8.8526 | Train Loss Unweighted: 3.0753 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7497 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1396 | Regression_eta: 0.0079 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7394 | Classification_binary: 2.7497 | Regression_cos_phi: 0.4273 | Regression_energy: 1.8789 | Regression_eta: 0.3309 | Regression_pt: 2.0456 | Regression_sin_phi: 0.6808 | LR: 7.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.1107 | max_backward: 0.0334 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3663 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3992.5 | uss_MB: 3514.1 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:08:48,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,518] INFO : Step 18000/50000 rank0 | Train Loss: 8.8607 | Train Loss Unweighted: 2.9714 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6479 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1377 | Regression_eta: 0.0092 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7005 | Classification_binary: 2.6479 | Regression_cos_phi: 0.5440 | Regression_energy: 1.8541 | Regression_eta: 0.3863 | Regression_pt: 1.9991 | Regression_sin_phi: 0.7288 | LR: 7.42e-04 | DataLoad Time: 0.0174s | Model Forward Time: 1.3215s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0666 | max_backward: 0.0337 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3991.9 | uss_MB: 3512.9 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:01,728] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,745] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 12:09:01,825] INFO : Running validation on rank0 +rank0 [2026-10-28 12:09:01,825] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:09:01,825] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:09:02,909] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 12:09:03,054] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:09:03,185] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:09:03,260] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:09:03,328] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:09:03,397] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:09:03,468] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:09:03,539] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:09:03,650] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:09:03,774] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:09:03,896] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:09:03,967] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:09:04,037] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:09:04,104] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:09:04,174] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:09:04,244] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:09:04,348] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:09:04,354] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:09:04,357] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:09:04,358] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:09:04,361] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:09:04,363] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:09:04,364] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:09:04,367] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:09:04,370] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:09:04,373] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:09:04,374] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:09:04,377] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:09:04,379] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:09:04,380] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:09:04,383] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:09:04,385] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:09:10,200] INFO : VALIDATION | Step=18000/50000 | Train Loss=8.8607 | Valid Loss=9.4343 | Train Loss Unweighted=2.9714 | Valid Loss Unweighted=3.1684 | Stale=0 | ETA=100.6m +rank0 [2026-10-28 12:09:10,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:10,270] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:09:10,274] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:10,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:11,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:12,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:12,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:13,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:13,686] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:13,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:13,745] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:09:13,755] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:13,914] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:15,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:15,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:16,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:17,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:17,177] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:17,180] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:17,235] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:09:17,250] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:09:17,409] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:18,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:19,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:19,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:20,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:20,525] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:20,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:20,780] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:21,235] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:22,672] INFO : Plotted jets +rank0 [2026-10-28 12:09:23,755] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:23,787] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:24,010] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:24,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:26,219] INFO : Plotted jets +rank0 [2026-10-28 12:09:27,329] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:27,362] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 12:09:27,561] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 12:09:27,979] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:29,375] INFO : Plotted jets +rank0 [2026-10-28 12:09:30,470] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:43,407] INFO : Step 18100/50000 rank0 | Train Loss: 9.9364 | Train Loss Unweighted: 3.1389 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7653 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1598 | Regression_eta: 0.0134 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7614 | Classification_binary: 2.7653 | Regression_cos_phi: 0.6029 | Regression_energy: 2.1516 | Regression_eta: 0.5602 | Regression_pt: 2.2930 | Regression_sin_phi: 0.8019 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0033 | max_backward: 0.0910 | max_optimizer_step: 0.1168 | max_tensorboard_logging: 0.2707 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4188.7 | uss_MB: 3711.5 | child_rss_MB: 3205.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:43,621] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:56,388] INFO : Step 18200/50000 rank0 | Train Loss: 9.0530 | Train Loss Unweighted: 3.2237 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.8909 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1443 | Regression_eta: 0.0088 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7586 | Classification_binary: 2.8909 | Regression_cos_phi: 0.4059 | Regression_energy: 1.9427 | Regression_eta: 0.3678 | Regression_pt: 2.0795 | Regression_sin_phi: 0.6076 | LR: 7.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 1.2574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0442 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.3196 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4188.7 | uss_MB: 3711.5 | child_rss_MB: 3203.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:56,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:09,669] INFO : Step 18300/50000 rank0 | Train Loss: 9.5721 | Train Loss Unweighted: 3.1719 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.8116 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1537 | Regression_eta: 0.0112 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7762 | Classification_binary: 2.8116 | Regression_cos_phi: 0.4728 | Regression_energy: 2.0694 | Regression_eta: 0.4709 | Regression_pt: 2.2647 | Regression_sin_phi: 0.7065 | LR: 7.34e-04 | DataLoad Time: 0.0414s | Model Forward Time: 1.3533s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1027 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3736 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4188.7 | uss_MB: 3711.6 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:10:09,882] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:22,699] INFO : Step 18400/50000 rank0 | Train Loss: 8.8849 | Train Loss Unweighted: 3.0690 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7434 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1379 | Regression_eta: 0.0087 | Regression_pt: 0.1401 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7510 | Classification_binary: 2.7434 | Regression_cos_phi: 0.4559 | Regression_energy: 1.8563 | Regression_eta: 0.3640 | Regression_pt: 2.0494 | Regression_sin_phi: 0.6650 | LR: 7.31e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1984s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0603 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4066.5 | uss_MB: 3587.8 | child_rss_MB: 3220.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:10:22,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:35,352] INFO : Step 18500/50000 rank0 | Train Loss: 9.3257 | Train Loss Unweighted: 3.0952 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.7397 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1550 | Regression_eta: 0.0093 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.7397 | Regression_cos_phi: 0.4608 | Regression_energy: 2.0865 | Regression_eta: 0.3902 | Regression_pt: 2.2286 | Regression_sin_phi: 0.6868 | LR: 7.29e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.1864s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0552 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.2791 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4066.5 | uss_MB: 3588.6 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:10:35,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:48,600] INFO : Step 18600/50000 rank0 | Train Loss: 10.0828 | Train Loss Unweighted: 3.1080 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7266 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1625 | Regression_eta: 0.0116 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7937 | Classification_binary: 2.7266 | Regression_cos_phi: 0.6805 | Regression_energy: 2.1880 | Regression_eta: 0.4868 | Regression_pt: 2.3474 | Regression_sin_phi: 0.8598 | LR: 7.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2675s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4066.5 | uss_MB: 3588.8 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:10:48,828] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:01,888] INFO : Step 18700/50000 rank0 | Train Loss: 9.3771 | Train Loss Unweighted: 3.1947 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8605 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1403 | Regression_eta: 0.0113 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7263 | Classification_binary: 2.8605 | Regression_cos_phi: 0.6075 | Regression_energy: 1.8888 | Regression_eta: 0.4715 | Regression_pt: 2.0525 | Regression_sin_phi: 0.7700 | LR: 7.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2469s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1937 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.5 | uss_MB: 3588.9 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:02,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:14,939] INFO : Step 18800/50000 rank0 | Train Loss: 9.3665 | Train Loss Unweighted: 3.1102 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7613 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1485 | Regression_eta: 0.0097 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7548 | Classification_binary: 2.7613 | Regression_cos_phi: 0.5850 | Regression_energy: 1.9993 | Regression_eta: 0.4084 | Regression_pt: 2.1793 | Regression_sin_phi: 0.6785 | LR: 7.21e-04 | DataLoad Time: 0.0830s | Model Forward Time: 1.2769s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0635 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3269 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4066.5 | uss_MB: 3588.9 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:15,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:27,792] INFO : Step 18900/50000 rank0 | Train Loss: 9.6127 | Train Loss Unweighted: 3.1807 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.8064 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1628 | Regression_eta: 0.0083 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8024 | Classification_binary: 2.8064 | Regression_cos_phi: 0.4683 | Regression_energy: 2.1915 | Regression_eta: 0.3492 | Regression_pt: 2.3881 | Regression_sin_phi: 0.6068 | LR: 7.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.2809 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4066.5 | uss_MB: 3589.0 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:28,024] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:40,466] INFO : Step 19000/50000 rank0 | Train Loss: 8.7202 | Train Loss Unweighted: 3.0424 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7234 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1366 | Regression_eta: 0.0099 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7017 | Classification_binary: 2.7234 | Regression_cos_phi: 0.4923 | Regression_energy: 1.8383 | Regression_eta: 0.4131 | Regression_pt: 1.9853 | Regression_sin_phi: 0.5662 | LR: 7.16e-04 | DataLoad Time: 0.0238s | Model Forward Time: 1.2594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0167 | max_loss: 0.0035 | max_backward: 0.1196 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.3 | uss_MB: 3588.0 | child_rss_MB: 3224.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:40,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:40,696] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 12:11:40,776] INFO : Running validation on rank0 +rank0 [2026-10-28 12:11:40,776] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:11:40,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:11:41,958] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 12:11:42,101] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:11:42,234] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:11:42,307] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:11:42,375] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:11:42,443] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:11:42,513] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:11:42,582] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:11:42,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:11:42,819] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:11:42,942] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:11:43,014] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:11:43,083] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:11:43,152] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:11:43,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:11:43,291] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:11:43,396] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:11:43,402] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:11:43,405] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:11:43,408] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:11:43,409] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:11:43,412] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:11:43,415] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:11:43,416] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:11:43,418] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:11:43,421] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:11:43,422] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:11:43,424] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:11:43,427] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:11:43,428] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:11:43,431] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:11:43,434] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:11:49,368] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.7202 | Valid Loss=9.4453 | Train Loss Unweighted=3.0424 | Valid Loss Unweighted=3.1741 | Stale=1 | ETA=96.7m +rank0 [2026-10-28 12:11:49,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:49,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:11:49,442] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:11:49,605] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:51,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:51,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:52,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:52,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:52,893] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:52,896] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:52,952] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:11:52,955] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:11:53,123] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:54,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:55,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:55,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:56,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:56,388] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:56,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:56,444] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:11:56,447] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:11:56,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:58,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:58,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:59,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:59,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:59,888] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:59,896] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:12:00,146] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:12:00,596] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:02,045] INFO : Plotted jets +rank0 [2026-10-28 12:12:03,139] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:03,171] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:03,391] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:03,772] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:05,618] INFO : Plotted jets +rank0 [2026-10-28 12:12:06,740] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:06,773] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 12:12:06,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 12:12:07,395] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:08,797] INFO : Plotted jets +rank0 [2026-10-28 12:12:09,900] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:22,827] INFO : Step 19100/50000 rank0 | Train Loss: 9.4328 | Train Loss Unweighted: 3.2166 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.8717 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1465 | Regression_eta: 0.0122 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7565 | Classification_binary: 2.8717 | Regression_cos_phi: 0.5552 | Regression_energy: 1.9720 | Regression_eta: 0.5094 | Regression_pt: 2.1309 | Regression_sin_phi: 0.6370 | LR: 7.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3042s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0452 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3712 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4270.0 | uss_MB: 3789.6 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:12:23,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:35,512] INFO : Step 19200/50000 rank0 | Train Loss: 9.4367 | Train Loss Unweighted: 3.0768 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.7381 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1409 | Regression_eta: 0.0129 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8070 | Classification_binary: 2.7381 | Regression_cos_phi: 0.5426 | Regression_energy: 1.8970 | Regression_eta: 0.5399 | Regression_pt: 2.0523 | Regression_sin_phi: 0.8597 | LR: 7.10e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3018s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0951 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0314 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3159 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4270.0 | uss_MB: 3789.6 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:12:35,728] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:12:48,453] INFO : Step 19300/50000 rank0 | Train Loss: 9.1221 | Train Loss Unweighted: 3.0047 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.6678 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1413 | Regression_eta: 0.0105 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7306 | Classification_binary: 2.6678 | Regression_cos_phi: 0.5114 | Regression_energy: 1.9026 | Regression_eta: 0.4405 | Regression_pt: 2.1163 | Regression_sin_phi: 0.7529 | LR: 7.08e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2735s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4270.0 | uss_MB: 3789.7 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:12:48,680] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:01,121] INFO : Step 19400/50000 rank0 | Train Loss: 8.5605 | Train Loss Unweighted: 3.0426 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.7370 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1290 | Regression_eta: 0.0105 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7124 | Classification_binary: 2.7370 | Regression_cos_phi: 0.3928 | Regression_energy: 1.7367 | Regression_eta: 0.4418 | Regression_pt: 1.8965 | Regression_sin_phi: 0.6433 | LR: 7.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.1373 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4270.0 | uss_MB: 3789.8 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:01,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:13,999] INFO : Step 19500/50000 rank0 | Train Loss: 10.2405 | Train Loss Unweighted: 3.1777 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7851 | Regression_cos_phi: 0.0122 | Regression_energy: 0.1684 | Regression_eta: 0.0106 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7583 | Classification_binary: 2.7851 | Regression_cos_phi: 0.6648 | Regression_energy: 2.2677 | Regression_eta: 0.4437 | Regression_pt: 2.4596 | Regression_sin_phi: 0.8612 | LR: 7.02e-04 | DataLoad Time: 0.0663s | Model Forward Time: 1.3521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1108 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1146 | max_tensorboard_logging: 0.3726 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4143.7 | uss_MB: 3663.2 | child_rss_MB: 3198.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:14,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:26,591] INFO : Step 19600/50000 rank0 | Train Loss: 9.0382 | Train Loss Unweighted: 2.9997 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6659 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1417 | Regression_eta: 0.0098 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7824 | Classification_binary: 2.6659 | Regression_cos_phi: 0.4995 | Regression_energy: 1.9076 | Regression_eta: 0.4112 | Regression_pt: 2.0644 | Regression_sin_phi: 0.7073 | LR: 7.00e-04 | DataLoad Time: 0.0732s | Model Forward Time: 1.3544s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1171 | max_loss: 0.0036 | max_backward: 0.0329 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:13:26,805] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:39,177] INFO : Step 19700/50000 rank0 | Train Loss: 8.5923 | Train Loss Unweighted: 2.8780 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5702 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1287 | Regression_eta: 0.0112 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6922 | Classification_binary: 2.5702 | Regression_cos_phi: 0.5372 | Regression_energy: 1.7331 | Regression_eta: 0.4700 | Regression_pt: 1.8823 | Regression_sin_phi: 0.7074 | LR: 6.97e-04 | DataLoad Time: 0.0472s | Model Forward Time: 1.3205s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0793 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:39,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:52,076] INFO : Step 19800/50000 rank0 | Train Loss: 9.1250 | Train Loss Unweighted: 2.9402 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.5984 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1453 | Regression_eta: 0.0118 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7823 | Classification_binary: 2.5984 | Regression_cos_phi: 0.5030 | Regression_energy: 1.9567 | Regression_eta: 0.4932 | Regression_pt: 2.1013 | Regression_sin_phi: 0.6901 | LR: 6.94e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1352 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3199.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:13:52,304] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:04,726] INFO : Step 19900/50000 rank0 | Train Loss: 9.8769 | Train Loss Unweighted: 3.0803 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7108 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1573 | Regression_eta: 0.0106 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7161 | Classification_binary: 2.7108 | Regression_cos_phi: 0.5994 | Regression_energy: 2.1177 | Regression_eta: 0.4462 | Regression_pt: 2.2940 | Regression_sin_phi: 0.9925 | LR: 6.92e-04 | DataLoad Time: 0.0143s | Model Forward Time: 1.1956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0558 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3206.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:04,942] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:17,249] INFO : Step 20000/50000 rank0 | Train Loss: 9.1370 | Train Loss Unweighted: 3.1946 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.8668 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1393 | Regression_eta: 0.0101 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8382 | Classification_binary: 2.8668 | Regression_cos_phi: 0.4600 | Regression_energy: 1.8757 | Regression_eta: 0.4251 | Regression_pt: 1.9999 | Regression_sin_phi: 0.6713 | LR: 6.89e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0171 | max_loss: 0.0033 | max_backward: 0.0389 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4144.7 | uss_MB: 3666.4 | child_rss_MB: 3206.1 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:14:17,477] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:17,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 12:14:17,576] INFO : Running validation on rank0 +rank0 [2026-10-28 12:14:17,576] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:14:17,576] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:14:18,742] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 12:14:18,887] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:14:19,019] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:14:19,093] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:14:19,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:14:19,228] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:14:19,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:14:19,368] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:14:19,481] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:14:19,604] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:14:19,725] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:14:19,796] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:14:19,864] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:14:19,932] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:14:20,001] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:14:20,071] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:14:20,172] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:14:20,178] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:14:20,181] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:14:20,184] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:14:20,185] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:14:20,187] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:14:20,189] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:14:20,191] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:14:20,194] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:14:20,195] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:14:20,198] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:14:20,201] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:14:20,204] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:14:20,207] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:14:20,208] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:14:20,211] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:14:26,088] INFO : VALIDATION | Step=20000/50000 | Train Loss=9.1370 | Valid Loss=9.3028 | Train Loss Unweighted=3.1946 | Valid Loss Unweighted=3.1245 | Stale=0 | ETA=92.8m +rank0 [2026-10-28 12:14:26,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:26,159] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:14:26,163] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:26,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:27,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:28,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:28,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:29,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:29,686] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:14:29,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:29,744] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:14:29,747] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:29,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:31,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:31,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:32,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:33,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:33,209] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:14:33,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:33,266] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:14:33,269] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:14:33,435] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:34,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:35,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:35,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:36,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:36,590] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:14:36,597] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:36,847] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:37,302] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:38,743] INFO : Plotted jets +rank0 [2026-10-28 12:14:39,848] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:39,881] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:40,104] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:40,481] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:42,363] INFO : Plotted jets +rank0 [2026-10-28 12:14:43,485] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:43,519] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 12:14:43,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 12:14:44,140] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:45,539] INFO : Plotted jets +rank0 [2026-10-28 12:14:46,642] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:59,276] INFO : Step 20100/50000 rank0 | Train Loss: 9.4646 | Train Loss Unweighted: 3.0714 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7355 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1364 | Regression_eta: 0.0213 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 2.7355 | Regression_cos_phi: 0.4837 | Regression_energy: 1.8365 | Regression_eta: 0.8923 | Regression_pt: 2.0015 | Regression_sin_phi: 0.7256 | LR: 6.86e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3609s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0799 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4343.8 | uss_MB: 3864.9 | child_rss_MB: 3204.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:14:59,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:11,784] INFO : Step 20200/50000 rank0 | Train Loss: 8.9915 | Train Loss Unweighted: 3.0969 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7853 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1306 | Regression_eta: 0.0109 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6943 | Classification_binary: 2.7853 | Regression_cos_phi: 0.5531 | Regression_energy: 1.7587 | Regression_eta: 0.4580 | Regression_pt: 1.8797 | Regression_sin_phi: 0.8625 | LR: 6.84e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4078s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0959 | max_backward: 0.0338 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4343.8 | uss_MB: 3864.9 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:15:11,994] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:24,592] INFO : Step 20300/50000 rank0 | Train Loss: 9.0444 | Train Loss Unweighted: 3.0223 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.6928 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1397 | Regression_eta: 0.0095 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7550 | Classification_binary: 2.6928 | Regression_cos_phi: 0.4464 | Regression_energy: 1.8804 | Regression_eta: 0.3971 | Regression_pt: 2.0394 | Regression_sin_phi: 0.8333 | LR: 6.81e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2209s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1189 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 4343.8 | uss_MB: 3865.0 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:15:24,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:37,088] INFO : Step 20400/50000 rank0 | Train Loss: 9.0596 | Train Loss Unweighted: 2.9935 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.6669 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1368 | Regression_eta: 0.0117 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7242 | Classification_binary: 2.6669 | Regression_cos_phi: 0.5375 | Regression_energy: 1.8413 | Regression_eta: 0.4904 | Regression_pt: 2.0015 | Regression_sin_phi: 0.7978 | LR: 6.78e-04 | DataLoad Time: 0.0368s | Model Forward Time: 1.3194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0570 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.3290 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4343.8 | uss_MB: 3865.0 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:37,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:49,873] INFO : Step 20500/50000 rank0 | Train Loss: 8.6218 | Train Loss Unweighted: 3.0504 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7346 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1348 | Regression_eta: 0.0092 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7010 | Classification_binary: 2.7346 | Regression_cos_phi: 0.3867 | Regression_energy: 1.8145 | Regression_eta: 0.3875 | Regression_pt: 1.9940 | Regression_sin_phi: 0.6036 | LR: 6.76e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4215.4 | uss_MB: 3736.5 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:15:50,099] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:02,226] INFO : Step 20600/50000 rank0 | Train Loss: 9.1607 | Train Loss Unweighted: 3.0830 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7366 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1491 | Regression_eta: 0.0091 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7477 | Classification_binary: 2.7366 | Regression_cos_phi: 0.4726 | Regression_energy: 2.0069 | Regression_eta: 0.3830 | Regression_pt: 2.1872 | Regression_sin_phi: 0.6268 | LR: 6.73e-04 | DataLoad Time: 0.0446s | Model Forward Time: 1.2999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0670 | max_forward: 0.0189 | max_loss: 0.0034 | max_backward: 0.0389 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3235 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4216.4 | uss_MB: 3739.0 | child_rss_MB: 3205.9 | allocated_MB: 210.7 | reserved_MB: 2802.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:16:02,441] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:14,463] INFO : Step 20700/50000 rank0 | Train Loss: 10.0484 | Train Loss Unweighted: 3.1710 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.7894 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1622 | Regression_eta: 0.0113 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7878 | Classification_binary: 2.7894 | Regression_cos_phi: 0.6495 | Regression_energy: 2.1830 | Regression_eta: 0.4722 | Regression_pt: 2.3866 | Regression_sin_phi: 0.7799 | LR: 6.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2282s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0825 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4216.4 | uss_MB: 3737.0 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:16:14,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:26,727] INFO : Step 20800/50000 rank0 | Train Loss: 9.8466 | Train Loss Unweighted: 3.1431 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.7624 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1640 | Regression_eta: 0.0121 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8033 | Classification_binary: 2.7624 | Regression_cos_phi: 0.5079 | Regression_energy: 2.2074 | Regression_eta: 0.5054 | Regression_pt: 2.3848 | Regression_sin_phi: 0.6754 | LR: 6.67e-04 | DataLoad Time: 0.0580s | Model Forward Time: 1.3640s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0729 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.3737 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4216.4 | uss_MB: 3737.1 | child_rss_MB: 3210.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:16:26,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:38,924] INFO : Step 20900/50000 rank0 | Train Loss: 9.2803 | Train Loss Unweighted: 3.0268 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6909 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1425 | Regression_eta: 0.0117 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7879 | Classification_binary: 2.6909 | Regression_cos_phi: 0.4520 | Regression_energy: 1.9190 | Regression_eta: 0.4896 | Regression_pt: 2.0268 | Regression_sin_phi: 0.9140 | LR: 6.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2160s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0319 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4216.4 | uss_MB: 3737.2 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:39,150] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:51,124] INFO : Step 21000/50000 rank0 | Train Loss: 9.1252 | Train Loss Unweighted: 3.1063 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7745 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1405 | Regression_eta: 0.0103 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7180 | Classification_binary: 2.7745 | Regression_cos_phi: 0.4627 | Regression_energy: 1.8915 | Regression_eta: 0.4300 | Regression_pt: 2.0698 | Regression_sin_phi: 0.7786 | LR: 6.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3654s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0824 | max_backward: 0.0328 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3678 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4216.4 | uss_MB: 3736.1 | child_rss_MB: 3209.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:16:51,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:51,364] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 12:16:51,549] INFO : Running validation on rank0 +rank0 [2026-10-28 12:16:51,549] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:16:51,549] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:16:52,608] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 12:16:52,751] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:16:52,882] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:16:52,956] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:16:53,025] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:16:53,094] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:16:53,164] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:16:53,234] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:16:53,346] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:16:53,470] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:16:53,592] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:16:53,665] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:16:53,736] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:16:53,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:16:53,875] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:16:53,944] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:16:54,055] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:16:54,061] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:16:54,064] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:16:54,067] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:16:54,068] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:16:54,071] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:16:54,074] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:16:54,075] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:16:54,078] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:16:54,081] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:16:54,083] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:16:54,084] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:16:54,087] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:16:54,090] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:16:54,093] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:16:54,094] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:16:59,887] INFO : VALIDATION | Step=21000/50000 | Train Loss=9.1252 | Valid Loss=9.3027 | Train Loss Unweighted=3.1063 | Valid Loss Unweighted=3.0990 | Stale=0 | ETA=89.0m +rank0 [2026-10-28 12:16:59,897] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:16:59,960] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:16:59,963] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:00,126] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:01,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:02,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:02,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:03,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:03,434] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:03,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:03,494] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:17:03,497] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:03,590] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:05,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:05,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:06,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:06,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:06,820] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:06,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:06,878] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:17:06,881] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:17:07,044] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:08,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:08,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:09,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:09,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:10,122] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:10,129] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:10,380] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:10,836] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:12,279] INFO : Plotted jets +rank0 [2026-10-28 12:17:13,366] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:13,398] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:13,623] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:14,001] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:15,849] INFO : Plotted jets +rank0 [2026-10-28 12:17:16,965] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:16,999] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:17,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:17,642] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:19,041] INFO : Plotted jets +rank0 [2026-10-28 12:17:20,131] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:33,068] INFO : Step 21100/50000 rank0 | Train Loss: 9.5936 | Train Loss Unweighted: 3.2061 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.8439 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1565 | Regression_eta: 0.0081 | Regression_pt: 0.1571 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7401 | Classification_binary: 2.8439 | Regression_cos_phi: 0.4770 | Regression_energy: 2.1066 | Regression_eta: 0.3377 | Regression_pt: 2.2970 | Regression_sin_phi: 0.7913 | LR: 6.59e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.0326 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4415.3 | uss_MB: 3934.8 | child_rss_MB: 3207.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:33,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:34,145] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 12:17:34,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 12:17:44,300] INFO : Step 21200/50000 rank0 | Train Loss: 9.0116 | Train Loss Unweighted: 3.0237 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6884 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1431 | Regression_eta: 0.0102 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7705 | Classification_binary: 2.6884 | Regression_cos_phi: 0.4648 | Regression_energy: 1.9264 | Regression_eta: 0.4290 | Regression_pt: 2.0858 | Regression_sin_phi: 0.6468 | LR: 6.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.1190 | max_optimizer_step: 0.1153 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4453.5 | uss_MB: 3973.1 | child_rss_MB: 3228.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:44,512] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:55,180] INFO : Step 21300/50000 rank0 | Train Loss: 9.2374 | Train Loss Unweighted: 3.0757 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7334 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1470 | Regression_eta: 0.0108 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.7334 | Regression_cos_phi: 0.4271 | Regression_energy: 1.9792 | Regression_eta: 0.4537 | Regression_pt: 2.1026 | Regression_sin_phi: 0.7488 | LR: 6.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0137 | Memory: rss_MB: 4453.5 | uss_MB: 3973.1 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:55,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:06,147] INFO : Step 21400/50000 rank0 | Train Loss: 9.2566 | Train Loss Unweighted: 3.0260 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6866 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1424 | Regression_eta: 0.0124 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7671 | Classification_binary: 2.6866 | Regression_cos_phi: 0.4784 | Regression_energy: 1.9167 | Regression_eta: 0.5188 | Regression_pt: 2.0948 | Regression_sin_phi: 0.7942 | LR: 6.51e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0331 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4453.5 | uss_MB: 3973.2 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:06,360] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:17,043] INFO : Step 21500/50000 rank0 | Train Loss: 8.8602 | Train Loss Unweighted: 3.0322 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.7125 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1361 | Regression_eta: 0.0107 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7281 | Classification_binary: 2.7125 | Regression_cos_phi: 0.4642 | Regression_energy: 1.8321 | Regression_eta: 0.4498 | Regression_pt: 1.9575 | Regression_sin_phi: 0.7160 | LR: 6.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0529 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3174 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4286.0 | uss_MB: 3806.1 | child_rss_MB: 3243.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:17,258] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:28,094] INFO : Step 21600/50000 rank0 | Train Loss: 8.9754 | Train Loss Unweighted: 2.9932 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6670 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1374 | Regression_eta: 0.0094 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.6670 | Regression_cos_phi: 0.5391 | Regression_energy: 1.8494 | Regression_eta: 0.3932 | Regression_pt: 2.0281 | Regression_sin_phi: 0.7934 | LR: 6.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2913s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0031 | max_backward: 0.1367 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4287.0 | uss_MB: 3807.1 | child_rss_MB: 3211.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:28,309] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:39,155] INFO : Step 21700/50000 rank0 | Train Loss: 8.6080 | Train Loss Unweighted: 2.9433 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6211 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1387 | Regression_eta: 0.0084 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6917 | Classification_binary: 2.6211 | Regression_cos_phi: 0.4209 | Regression_energy: 1.8674 | Regression_eta: 0.3504 | Regression_pt: 2.0353 | Regression_sin_phi: 0.6211 | LR: 6.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2494s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0157 | max_loss: 0.0030 | max_backward: 0.0343 | max_optimizer_step: 0.1487 | max_tensorboard_logging: 0.3293 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4287.0 | uss_MB: 3807.1 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:18:39,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:50,021] INFO : Step 21800/50000 rank0 | Train Loss: 8.9894 | Train Loss Unweighted: 3.0161 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6833 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1428 | Regression_eta: 0.0088 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6643 | Classification_binary: 2.6833 | Regression_cos_phi: 0.4654 | Regression_energy: 1.9226 | Regression_eta: 0.3671 | Regression_pt: 2.0910 | Regression_sin_phi: 0.7956 | LR: 6.40e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3319s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1020 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3375 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4287.0 | uss_MB: 3807.2 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:50,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:01,103] INFO : Step 21900/50000 rank0 | Train Loss: 9.0555 | Train Loss Unweighted: 3.0828 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7568 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1385 | Regression_eta: 0.0126 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.7568 | Regression_cos_phi: 0.4308 | Regression_energy: 1.8645 | Regression_eta: 0.5273 | Regression_pt: 1.9831 | Regression_sin_phi: 0.7458 | LR: 6.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3044s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0782 | max_forward: 0.0164 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3238 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4287.0 | uss_MB: 3807.2 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:01,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:12,245] INFO : Step 22000/50000 rank0 | Train Loss: 9.7307 | Train Loss Unweighted: 3.1206 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.7446 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1626 | Regression_eta: 0.0106 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8043 | Classification_binary: 2.7446 | Regression_cos_phi: 0.4527 | Regression_energy: 2.1889 | Regression_eta: 0.4450 | Regression_pt: 2.3614 | Regression_sin_phi: 0.7338 | LR: 6.34e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3335s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1019 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.3273 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4287.0 | uss_MB: 3807.3 | child_rss_MB: 3217.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:12,459] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:12,502] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 12:19:12,583] INFO : Running validation on rank0 +rank0 [2026-10-28 12:19:12,583] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:19:12,583] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:19:13,674] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 12:19:13,817] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:19:13,948] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:19:14,023] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:19:14,092] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:19:14,160] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:19:14,230] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:19:14,300] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:19:14,411] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:19:14,536] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:19:14,658] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:19:14,732] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:19:14,802] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:19:14,871] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:19:14,940] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:19:15,018] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:19:15,138] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:19:15,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:19:15,147] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:19:15,150] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:19:15,151] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:19:15,154] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:19:15,157] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:19:15,159] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:19:15,160] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:19:15,163] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:19:15,164] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:19:15,167] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:19:15,169] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:19:15,170] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:19:15,173] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:19:15,176] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:19:20,990] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.7307 | Valid Loss=9.1295 | Train Loss Unweighted=3.1206 | Valid Loss Unweighted=3.0547 | Stale=0 | ETA=85.0m +rank0 [2026-10-28 12:19:21,002] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:21,067] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:19:21,071] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:21,237] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:22,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:23,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:23,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:24,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:24,424] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:24,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:24,485] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:19:24,487] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:24,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:25,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:26,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:27,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:27,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:27,868] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:27,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:27,925] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:19:27,928] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:19:28,039] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:29,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:29,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:30,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:31,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:31,208] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:31,216] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,468] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,921] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:33,364] INFO : Plotted jets +rank0 [2026-10-28 12:19:34,446] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:34,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:34,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:35,087] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:36,942] INFO : Plotted jets +rank0 [2026-10-28 12:19:38,050] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:38,082] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 12:19:38,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 12:19:38,701] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:40,094] INFO : Plotted jets +rank0 [2026-10-28 12:19:41,184] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:52,415] INFO : Step 22100/50000 rank0 | Train Loss: 8.3414 | Train Loss Unweighted: 2.9058 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5959 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1329 | Regression_eta: 0.0083 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7125 | Classification_binary: 2.5959 | Regression_cos_phi: 0.3878 | Regression_energy: 1.7894 | Regression_eta: 0.3493 | Regression_pt: 1.9525 | Regression_sin_phi: 0.5540 | LR: 6.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3472s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0372 | max_backward: 0.0339 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4492.3 | uss_MB: 4013.6 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:19:52,625] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:03,570] INFO : Step 22200/50000 rank0 | Train Loss: 8.7378 | Train Loss Unweighted: 2.8832 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.5703 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1304 | Regression_eta: 0.0101 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7389 | Classification_binary: 2.5703 | Regression_cos_phi: 0.4395 | Regression_energy: 1.7561 | Regression_eta: 0.4237 | Regression_pt: 1.9188 | Regression_sin_phi: 0.8905 | LR: 6.29e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0133 | max_backward: 0.0328 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4492.3 | uss_MB: 4012.5 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:03,786] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:14,615] INFO : Step 22300/50000 rank0 | Train Loss: 8.9905 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6960 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1459 | Regression_eta: 0.0102 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7162 | Classification_binary: 2.6960 | Regression_cos_phi: 0.4007 | Regression_energy: 1.9642 | Regression_eta: 0.4273 | Regression_pt: 2.1343 | Regression_sin_phi: 0.6520 | LR: 6.26e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2323s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0567 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4492.3 | uss_MB: 4013.8 | child_rss_MB: 3240.9 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:14,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:25,877] INFO : Step 22400/50000 rank0 | Train Loss: 9.1353 | Train Loss Unweighted: 3.1363 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7949 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1482 | Regression_eta: 0.0085 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7663 | Classification_binary: 2.7949 | Regression_cos_phi: 0.4483 | Regression_energy: 1.9955 | Regression_eta: 0.3559 | Regression_pt: 2.1320 | Regression_sin_phi: 0.6424 | LR: 6.23e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3818s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0769 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3242 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4492.3 | uss_MB: 4012.7 | child_rss_MB: 3215.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:20:26,091] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:37,033] INFO : Step 22500/50000 rank0 | Train Loss: 8.5704 | Train Loss Unweighted: 3.0317 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.7279 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1295 | Regression_eta: 0.0092 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7336 | Classification_binary: 2.7279 | Regression_cos_phi: 0.4771 | Regression_energy: 1.7436 | Regression_eta: 0.3867 | Regression_pt: 1.8486 | Regression_sin_phi: 0.6527 | LR: 6.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0329 | max_backward: 0.0337 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.4008 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4363.9 | uss_MB: 3885.1 | child_rss_MB: 3213.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:37,247] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:48,131] INFO : Step 22600/50000 rank0 | Train Loss: 8.5830 | Train Loss Unweighted: 2.9290 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6138 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1350 | Regression_eta: 0.0099 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7174 | Classification_binary: 2.6138 | Regression_cos_phi: 0.4004 | Regression_energy: 1.8177 | Regression_eta: 0.4132 | Regression_pt: 1.9506 | Regression_sin_phi: 0.6700 | LR: 6.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3149s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0505 | max_backward: 0.0335 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4363.9 | uss_MB: 3884.7 | child_rss_MB: 3217.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:48,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:58,968] INFO : Step 22700/50000 rank0 | Train Loss: 9.3718 | Train Loss Unweighted: 3.0961 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7625 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1388 | Regression_eta: 0.0112 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7644 | Classification_binary: 2.7625 | Regression_cos_phi: 0.5496 | Regression_energy: 1.8689 | Regression_eta: 0.4684 | Regression_pt: 2.0369 | Regression_sin_phi: 0.9211 | LR: 6.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2124s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4363.9 | uss_MB: 3886.0 | child_rss_MB: 3215.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:20:59,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:10,068] INFO : Step 22800/50000 rank0 | Train Loss: 8.5136 | Train Loss Unweighted: 2.9794 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6731 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1288 | Regression_eta: 0.0078 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7835 | Classification_binary: 2.6731 | Regression_cos_phi: 0.4744 | Regression_energy: 1.7343 | Regression_eta: 0.3285 | Regression_pt: 1.9018 | Regression_sin_phi: 0.6180 | LR: 6.12e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1674 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4364.9 | uss_MB: 3886.4 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:10,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:20,981] INFO : Step 22900/50000 rank0 | Train Loss: 8.9764 | Train Loss Unweighted: 2.9652 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6183 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1516 | Regression_eta: 0.0113 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6862 | Classification_binary: 2.6183 | Regression_cos_phi: 0.4573 | Regression_energy: 2.0403 | Regression_eta: 0.4715 | Regression_pt: 2.1751 | Regression_sin_phi: 0.5277 | LR: 6.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1750s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4364.9 | uss_MB: 3886.5 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:21:21,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:32,021] INFO : Step 23000/50000 rank0 | Train Loss: 8.3048 | Train Loss Unweighted: 2.9012 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1219 | Regression_eta: 0.0111 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6837 | Classification_binary: 2.6128 | Regression_cos_phi: 0.5082 | Regression_energy: 1.6417 | Regression_eta: 0.4657 | Regression_pt: 1.7161 | Regression_sin_phi: 0.6767 | LR: 6.06e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3291s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3169 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4364.9 | uss_MB: 3886.5 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:21:32,234] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:32,290] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 12:21:32,371] INFO : Running validation on rank0 +rank0 [2026-10-28 12:21:32,371] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:21:32,371] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:21:33,453] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 12:21:33,598] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:21:33,731] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:21:33,807] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:21:33,876] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:21:33,944] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:21:34,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:21:34,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:21:34,197] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:21:34,321] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:21:34,443] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:21:34,516] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:21:34,587] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:21:34,656] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:21:34,725] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:21:34,794] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:21:34,906] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:21:34,913] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:21:34,916] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:21:34,919] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:21:34,920] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:21:34,923] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:21:34,926] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:21:34,927] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:21:34,929] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:21:34,932] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:21:34,933] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:21:34,936] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:21:34,937] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:21:34,940] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:21:34,942] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:21:34,945] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:21:40,985] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.3048 | Valid Loss=9.0555 | Train Loss Unweighted=2.9012 | Valid Loss Unweighted=3.0136 | Stale=0 | ETA=81.1m +rank0 [2026-10-28 12:21:40,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:41,057] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:21:41,061] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:41,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:42,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:43,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:43,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:44,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:44,377] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:44,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:44,435] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:21:44,438] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:44,535] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:45,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:46,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:46,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:47,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:47,748] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:47,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:47,806] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:21:47,808] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:21:47,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:49,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:50,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:50,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:51,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:51,167] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:51,176] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:51,429] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:51,893] INFO : Plotted number of elements +rank0 [2026-10-28 12:21:53,341] INFO : Plotted jets +rank0 [2026-10-28 12:21:54,437] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:54,471] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:54,696] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:55,075] INFO : Plotted number of elements +rank0 [2026-10-28 12:21:56,972] INFO : Plotted jets +rank0 [2026-10-28 12:21:58,086] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:58,119] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 12:21:58,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 12:21:58,739] INFO : Plotted number of elements +rank0 [2026-10-28 12:22:00,144] INFO : Plotted jets +rank0 [2026-10-28 12:22:01,243] INFO : Plotted jet ratio +rank0 [2026-10-28 12:22:12,647] INFO : Step 23100/50000 rank0 | Train Loss: 9.2078 | Train Loss Unweighted: 3.0558 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7110 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1469 | Regression_eta: 0.0095 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7018 | Classification_binary: 2.7110 | Regression_cos_phi: 0.4685 | Regression_energy: 1.9779 | Regression_eta: 0.3979 | Regression_pt: 2.1854 | Regression_sin_phi: 0.7653 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4161s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1206 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3213.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:12,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:23,646] INFO : Step 23200/50000 rank0 | Train Loss: 8.6996 | Train Loss Unweighted: 2.9659 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6479 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1353 | Regression_eta: 0.0104 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7499 | Classification_binary: 2.6479 | Regression_cos_phi: 0.4622 | Regression_energy: 1.8215 | Regression_eta: 0.4368 | Regression_pt: 1.9575 | Regression_sin_phi: 0.6238 | LR: 6.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3255s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0324 | max_backward: 0.0335 | max_optimizer_step: 0.1169 | max_tensorboard_logging: 0.3913 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4565.3 | uss_MB: 4088.9 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:23,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:34,858] INFO : Step 23300/50000 rank0 | Train Loss: 8.6422 | Train Loss Unweighted: 2.9491 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6257 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1395 | Regression_eta: 0.0091 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6754 | Classification_binary: 2.6257 | Regression_cos_phi: 0.5012 | Regression_energy: 1.8774 | Regression_eta: 0.3794 | Regression_pt: 2.0295 | Regression_sin_phi: 0.5535 | LR: 5.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1069 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:22:35,071] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:45,935] INFO : Step 23400/50000 rank0 | Train Loss: 8.7262 | Train Loss Unweighted: 2.8483 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5290 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1346 | Regression_eta: 0.0107 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7434 | Classification_binary: 2.5290 | Regression_cos_phi: 0.4939 | Regression_energy: 1.8114 | Regression_eta: 0.4504 | Regression_pt: 1.9552 | Regression_sin_phi: 0.7430 | LR: 5.95e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3905s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1197 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3211.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:46,148] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:56,954] INFO : Step 23500/50000 rank0 | Train Loss: 8.6694 | Train Loss Unweighted: 2.8873 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5754 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1314 | Regression_eta: 0.0134 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6989 | Classification_binary: 2.5754 | Regression_cos_phi: 0.4789 | Regression_energy: 1.7695 | Regression_eta: 0.5617 | Regression_pt: 1.8840 | Regression_sin_phi: 0.7009 | LR: 5.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.1005 | max_backward: 0.0336 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4565.3 | uss_MB: 4087.9 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:57,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:08,066] INFO : Step 23600/50000 rank0 | Train Loss: 9.3664 | Train Loss Unweighted: 3.2217 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.8772 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1477 | Regression_eta: 0.0086 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7312 | Classification_binary: 2.8772 | Regression_cos_phi: 0.5057 | Regression_energy: 1.9878 | Regression_eta: 0.3612 | Regression_pt: 2.1656 | Regression_sin_phi: 0.7378 | LR: 5.90e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3435s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0671 | max_backward: 0.0337 | max_optimizer_step: 0.1173 | max_tensorboard_logging: 0.3781 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4438.2 | uss_MB: 3961.3 | child_rss_MB: 3205.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:08,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:19,304] INFO : Step 23700/50000 rank0 | Train Loss: 8.4570 | Train Loss Unweighted: 2.9799 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6745 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1311 | Regression_eta: 0.0088 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7596 | Classification_binary: 2.6745 | Regression_cos_phi: 0.4234 | Regression_energy: 1.7654 | Regression_eta: 0.3706 | Regression_pt: 1.8684 | Regression_sin_phi: 0.5950 | LR: 5.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3945s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1182 | max_loss: 0.0031 | max_backward: 0.0376 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4438.2 | uss_MB: 3965.0 | child_rss_MB: 3207.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:23:19,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:30,304] INFO : Step 23800/50000 rank0 | Train Loss: 9.0452 | Train Loss Unweighted: 3.0326 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7007 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1417 | Regression_eta: 0.0097 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7380 | Classification_binary: 2.7007 | Regression_cos_phi: 0.4831 | Regression_energy: 1.9071 | Regression_eta: 0.4053 | Regression_pt: 2.0531 | Regression_sin_phi: 0.7579 | LR: 5.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0159 | max_loss: 0.0453 | max_backward: 0.0326 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4439.2 | uss_MB: 3962.0 | child_rss_MB: 3207.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:23:30,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:41,187] INFO : Step 23900/50000 rank0 | Train Loss: 9.0834 | Train Loss Unweighted: 3.0904 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7674 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1349 | Regression_eta: 0.0114 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7750 | Classification_binary: 2.7674 | Regression_cos_phi: 0.5516 | Regression_energy: 1.8158 | Regression_eta: 0.4785 | Regression_pt: 1.9690 | Regression_sin_phi: 0.7261 | LR: 5.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2007s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0029 | max_backward: 0.0767 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.2678 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4439.2 | uss_MB: 3963.2 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:41,402] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:52,324] INFO : Step 24000/50000 rank0 | Train Loss: 9.8334 | Train Loss Unweighted: 3.0472 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6680 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1626 | Regression_eta: 0.0113 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.6680 | Regression_cos_phi: 0.6280 | Regression_energy: 2.1890 | Regression_eta: 0.4720 | Regression_pt: 2.3717 | Regression_sin_phi: 0.7547 | LR: 5.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2600s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0032 | max_backward: 0.1086 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4439.2 | uss_MB: 3963.2 | child_rss_MB: 3232.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:23:52,538] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:52,557] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 12:23:52,638] INFO : Running validation on rank0 +rank0 [2026-10-28 12:23:52,638] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:23:52,638] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:23:53,822] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 12:23:53,967] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:23:54,099] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:23:54,174] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:23:54,243] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:23:54,310] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:23:54,379] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:23:54,449] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:23:54,561] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:23:54,684] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:23:54,807] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:23:54,878] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:23:54,947] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:23:55,015] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:23:55,085] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:23:55,155] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:23:55,270] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:23:55,277] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:23:55,279] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:23:55,282] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:23:55,283] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:23:55,286] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:23:55,289] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:23:55,290] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:23:55,292] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:23:55,295] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:23:55,296] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:23:55,298] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:23:55,300] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:23:55,302] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:23:55,305] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:23:55,308] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:24:01,241] INFO : VALIDATION | Step=24000/50000 | Train Loss=9.8334 | Valid Loss=9.0501 | Train Loss Unweighted=3.0472 | Valid Loss Unweighted=3.0260 | Stale=0 | ETA=77.4m +rank0 [2026-10-28 12:24:01,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:01,311] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:24:01,314] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:01,477] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:03,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:03,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:04,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:04,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:04,828] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:24:04,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:04,887] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:24:04,890] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:04,993] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:06,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:06,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:07,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:08,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:08,253] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:24:08,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:08,311] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:24:08,314] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:24:08,482] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:09,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:10,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:10,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:11,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:11,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:24:11,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:11,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:12,349] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:13,802] INFO : Plotted jets +rank0 [2026-10-28 12:24:14,902] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:14,935] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:15,160] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:15,539] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:17,435] INFO : Plotted jets +rank0 [2026-10-28 12:24:18,501] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:18,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 12:24:18,736] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 12:24:19,154] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:20,557] INFO : Plotted jets +rank0 [2026-10-28 12:24:21,657] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:33,033] INFO : Step 24100/50000 rank0 | Train Loss: 9.2924 | Train Loss Unweighted: 3.0318 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6919 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1442 | Regression_eta: 0.0119 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7849 | Classification_binary: 2.6919 | Regression_cos_phi: 0.5482 | Regression_energy: 1.9410 | Regression_eta: 0.5004 | Regression_pt: 2.0609 | Regression_sin_phi: 0.7651 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1048 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2971 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3206.5 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:24:33,242] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:44,123] INFO : Step 24200/50000 rank0 | Train Loss: 8.6057 | Train Loss Unweighted: 2.9468 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6434 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1278 | Regression_eta: 0.0118 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7199 | Classification_binary: 2.6434 | Regression_cos_phi: 0.4161 | Regression_energy: 1.7201 | Regression_eta: 0.4940 | Regression_pt: 1.8298 | Regression_sin_phi: 0.7825 | LR: 5.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3208s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0401 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1211 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:24:44,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:55,162] INFO : Step 24300/50000 rank0 | Train Loss: 9.6636 | Train Loss Unweighted: 3.1530 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.7882 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1577 | Regression_eta: 0.0117 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7853 | Classification_binary: 2.7882 | Regression_cos_phi: 0.5293 | Regression_energy: 2.1233 | Regression_eta: 0.4887 | Regression_pt: 2.2481 | Regression_sin_phi: 0.7008 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3168s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0161 | max_backward: 0.0337 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:24:55,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:06,303] INFO : Step 24400/50000 rank0 | Train Loss: 8.5365 | Train Loss Unweighted: 3.0744 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7745 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1262 | Regression_eta: 0.0106 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6516 | Classification_binary: 2.7745 | Regression_cos_phi: 0.4420 | Regression_energy: 1.6992 | Regression_eta: 0.4445 | Regression_pt: 1.8624 | Regression_sin_phi: 0.6623 | LR: 5.67e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0381 | max_backward: 0.0326 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3921 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:06,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:17,198] INFO : Step 24500/50000 rank0 | Train Loss: 8.5981 | Train Loss Unweighted: 2.8085 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.4983 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1298 | Regression_eta: 0.0107 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7910 | Classification_binary: 2.4983 | Regression_cos_phi: 0.6159 | Regression_energy: 1.7469 | Regression_eta: 0.4466 | Regression_pt: 1.8577 | Regression_sin_phi: 0.6417 | LR: 5.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0746 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3224.5 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:17,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:28,208] INFO : Step 24600/50000 rank0 | Train Loss: 9.3902 | Train Loss Unweighted: 3.0343 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6847 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1490 | Regression_eta: 0.0117 | Regression_pt: 0.1470 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7400 | Classification_binary: 2.6847 | Regression_cos_phi: 0.6020 | Regression_energy: 2.0056 | Regression_eta: 0.4907 | Regression_pt: 2.1504 | Regression_sin_phi: 0.7168 | LR: 5.61e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2991s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.2216 | max_tensorboard_logging: 0.2926 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4513.8 | uss_MB: 4036.5 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:25:28,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:39,332] INFO : Step 24700/50000 rank0 | Train Loss: 9.0457 | Train Loss Unweighted: 3.0486 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7218 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1373 | Regression_eta: 0.0092 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7666 | Classification_binary: 2.7218 | Regression_cos_phi: 0.5160 | Regression_energy: 1.8489 | Regression_eta: 0.3838 | Regression_pt: 2.0234 | Regression_sin_phi: 0.7853 | LR: 5.58e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4513.8 | uss_MB: 4036.6 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:25:39,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:50,205] INFO : Step 24800/50000 rank0 | Train Loss: 8.7281 | Train Loss Unweighted: 2.9290 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6053 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1372 | Regression_eta: 0.0110 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6748 | Classification_binary: 2.6053 | Regression_cos_phi: 0.5224 | Regression_energy: 1.8467 | Regression_eta: 0.4595 | Regression_pt: 2.0281 | Regression_sin_phi: 0.5912 | LR: 5.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2510s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0247 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0341 | max_optimizer_step: 0.1200 | max_tensorboard_logging: 0.3101 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4513.8 | uss_MB: 4036.3 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:25:50,420] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:01,290] INFO : Step 24900/50000 rank0 | Train Loss: 8.4344 | Train Loss Unweighted: 2.9049 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6030 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1267 | Regression_eta: 0.0095 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.6030 | Regression_cos_phi: 0.4542 | Regression_energy: 1.7052 | Regression_eta: 0.4000 | Regression_pt: 1.8677 | Regression_sin_phi: 0.6990 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1016 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4513.8 | uss_MB: 4036.3 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:26:01,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:12,274] INFO : Step 25000/50000 rank0 | Train Loss: 8.5499 | Train Loss Unweighted: 2.9449 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6413 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1283 | Regression_eta: 0.0102 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6905 | Classification_binary: 2.6413 | Regression_cos_phi: 0.5190 | Regression_energy: 1.7269 | Regression_eta: 0.4286 | Regression_pt: 1.8491 | Regression_sin_phi: 0.6945 | LR: 5.50e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3526s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0714 | max_backward: 0.0327 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.3685 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4513.8 | uss_MB: 4036.6 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:26:12,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:12,505] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 12:26:12,690] INFO : Running validation on rank0 +rank0 [2026-10-28 12:26:12,690] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:26:12,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:26:13,770] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 12:26:13,914] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:26:14,046] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:26:14,119] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:26:14,188] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:26:14,255] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:26:14,325] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:26:14,393] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:26:14,505] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:26:14,629] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:26:14,750] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:26:14,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:26:14,892] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:26:14,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:26:15,029] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:26:15,098] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:26:15,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:26:15,209] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:26:15,211] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:26:15,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:26:15,215] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:26:15,217] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:26:15,220] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:26:15,221] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:26:15,223] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:26:15,225] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:26:15,227] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:26:15,230] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:26:15,232] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:26:15,235] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:26:15,236] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:26:15,238] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:26:21,015] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.5499 | Valid Loss=8.9006 | Train Loss Unweighted=2.9449 | Valid Loss Unweighted=2.9816 | Stale=0 | ETA=73.8m +rank0 [2026-10-28 12:26:21,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:21,085] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:26:21,089] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:21,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:22,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:23,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:23,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:24,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:24,522] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:24,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:24,581] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:26:24,584] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:24,752] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:26,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:26,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:27,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:27,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:28,090] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:28,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:28,149] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:26:28,153] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:26:28,250] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:29,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:30,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:30,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:31,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:31,333] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:31,340] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:31,591] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:32,047] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:33,488] INFO : Plotted jets +rank0 [2026-10-28 12:26:34,594] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:34,627] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:34,857] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:35,233] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:37,116] INFO : Plotted jets +rank0 [2026-10-28 12:26:38,175] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:38,208] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 12:26:38,408] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 12:26:38,822] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:40,219] INFO : Plotted jets +rank0 [2026-10-28 12:26:41,312] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:52,539] INFO : Step 25100/50000 rank0 | Train Loss: 9.3452 | Train Loss Unweighted: 3.0107 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1528 | Regression_eta: 0.0102 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7839 | Classification_binary: 2.6540 | Regression_cos_phi: 0.4921 | Regression_energy: 2.0576 | Regression_eta: 0.4280 | Regression_pt: 2.2339 | Regression_sin_phi: 0.6957 | LR: 5.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0602 | max_backward: 0.0337 | max_optimizer_step: 0.1226 | max_tensorboard_logging: 0.3646 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:26:52,753] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:03,673] INFO : Step 25200/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.9470 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6397 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1300 | Regression_eta: 0.0102 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7365 | Classification_binary: 2.6397 | Regression_cos_phi: 0.4627 | Regression_energy: 1.7502 | Regression_eta: 0.4257 | Regression_pt: 1.8794 | Regression_sin_phi: 0.6661 | LR: 5.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3505s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0662 | max_backward: 0.0341 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3674 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:27:03,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:14,605] INFO : Step 25300/50000 rank0 | Train Loss: 8.5483 | Train Loss Unweighted: 2.9748 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6578 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1361 | Regression_eta: 0.0085 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.6578 | Regression_cos_phi: 0.4192 | Regression_energy: 1.8317 | Regression_eta: 0.3581 | Regression_pt: 1.9841 | Regression_sin_phi: 0.5501 | LR: 5.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0704 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3223.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:27:14,818] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:25,640] INFO : Step 25400/50000 rank0 | Train Loss: 9.0436 | Train Loss Unweighted: 3.0074 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6784 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1386 | Regression_eta: 0.0100 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7470 | Classification_binary: 2.6784 | Regression_cos_phi: 0.5386 | Regression_energy: 1.8661 | Regression_eta: 0.4172 | Regression_pt: 2.0311 | Regression_sin_phi: 0.7652 | LR: 5.39e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1986 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4714.0 | uss_MB: 4233.9 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:25,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:36,756] INFO : Step 25500/50000 rank0 | Train Loss: 8.6778 | Train Loss Unweighted: 2.9564 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6444 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1314 | Regression_eta: 0.0104 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.6444 | Regression_cos_phi: 0.4234 | Regression_energy: 1.7690 | Regression_eta: 0.4364 | Regression_pt: 1.9156 | Regression_sin_phi: 0.7391 | LR: 5.36e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0168 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4714.0 | uss_MB: 4233.9 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:36,984] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:47,713] INFO : Step 25600/50000 rank0 | Train Loss: 8.6983 | Train Loss Unweighted: 2.9770 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6611 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1340 | Regression_eta: 0.0110 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6931 | Classification_binary: 2.6611 | Regression_cos_phi: 0.4315 | Regression_energy: 1.8039 | Regression_eta: 0.4614 | Regression_pt: 1.9566 | Regression_sin_phi: 0.6908 | LR: 5.33e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2761 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4593.8 | uss_MB: 4112.3 | child_rss_MB: 3203.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:47,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:58,726] INFO : Step 25700/50000 rank0 | Train Loss: 9.0989 | Train Loss Unweighted: 3.0015 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6423 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1563 | Regression_eta: 0.0087 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7876 | Classification_binary: 2.6423 | Regression_cos_phi: 0.3569 | Regression_energy: 2.1037 | Regression_eta: 0.3628 | Regression_pt: 2.3054 | Regression_sin_phi: 0.5401 | LR: 5.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3663s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0802 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.3744 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3204.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:58,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:09,851] INFO : Step 25800/50000 rank0 | Train Loss: 8.9372 | Train Loss Unweighted: 2.8968 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.5623 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1428 | Regression_eta: 0.0104 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7489 | Classification_binary: 2.5623 | Regression_cos_phi: 0.5540 | Regression_energy: 1.9230 | Regression_eta: 0.4365 | Regression_pt: 2.0577 | Regression_sin_phi: 0.6547 | LR: 5.27e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3830s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0960 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:10,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:20,663] INFO : Step 25900/50000 rank0 | Train Loss: 9.5949 | Train Loss Unweighted: 3.1370 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7832 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1512 | Regression_eta: 0.0112 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7232 | Classification_binary: 2.7832 | Regression_cos_phi: 0.5369 | Regression_energy: 2.0358 | Regression_eta: 0.4698 | Regression_pt: 2.1842 | Regression_sin_phi: 0.8618 | LR: 5.25e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2329s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0658 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3224.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:28:20,876] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:31,696] INFO : Step 26000/50000 rank0 | Train Loss: 9.4804 | Train Loss Unweighted: 3.0324 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6751 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1532 | Regression_eta: 0.0113 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7788 | Classification_binary: 2.6751 | Regression_cos_phi: 0.5717 | Regression_energy: 2.0618 | Regression_eta: 0.4742 | Regression_pt: 2.1967 | Regression_sin_phi: 0.7220 | LR: 5.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2703s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.2709 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.5 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:28:31,925] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:31,941] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 12:28:32,021] INFO : Running validation on rank0 +rank0 [2026-10-28 12:28:32,021] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:28:32,021] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:28:33,202] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 12:28:33,352] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:28:33,484] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:28:33,558] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:28:33,627] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:28:33,695] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:28:33,765] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:28:33,834] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:28:33,948] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:28:34,073] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:28:34,196] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:28:34,268] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:28:34,338] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:28:34,406] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:28:34,475] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:28:34,543] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:28:34,653] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:28:34,659] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:28:34,662] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:28:34,665] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:28:34,668] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:28:34,669] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:28:34,672] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:28:34,672] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:28:34,675] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:28:34,678] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:28:34,679] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:28:34,682] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:28:34,684] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:28:34,685] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:28:34,688] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:28:34,691] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:28:40,639] INFO : VALIDATION | Step=26000/50000 | Train Loss=9.4804 | Valid Loss=8.8191 | Train Loss Unweighted=3.0324 | Valid Loss Unweighted=2.9569 | Stale=0 | ETA=70.3m +rank0 [2026-10-28 12:28:40,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:40,711] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:28:40,714] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:40,784] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:42,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:42,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:43,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:44,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:44,276] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:28:44,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:44,334] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:28:44,337] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:44,455] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:46,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:46,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:47,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:47,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:47,976] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:28:47,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:48,034] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:28:48,037] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:28:48,140] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:49,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:50,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:50,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:51,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:51,466] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:28:51,473] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:51,731] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:52,189] INFO : Plotted number of elements +rank0 [2026-10-28 12:28:53,634] INFO : Plotted jets +rank0 [2026-10-28 12:28:54,735] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:54,767] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:54,992] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:55,375] INFO : Plotted number of elements +rank0 [2026-10-28 12:28:57,255] INFO : Plotted jets +rank0 [2026-10-28 12:28:58,373] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:58,404] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 12:28:58,608] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 12:28:59,024] INFO : Plotted number of elements +rank0 [2026-10-28 12:29:00,425] INFO : Plotted jets +rank0 [2026-10-28 12:29:01,530] INFO : Plotted jet ratio +rank0 [2026-10-28 12:29:12,920] INFO : Step 26100/50000 rank0 | Train Loss: 9.3825 | Train Loss Unweighted: 2.9760 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6205 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1528 | Regression_eta: 0.0091 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7711 | Classification_binary: 2.6205 | Regression_cos_phi: 0.6339 | Regression_energy: 2.0566 | Regression_eta: 0.3833 | Regression_pt: 2.1951 | Regression_sin_phi: 0.7221 | LR: 5.19e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1103 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1221 | max_tensorboard_logging: 0.3668 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4789.6 | uss_MB: 4309.9 | child_rss_MB: 3204.0 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:29:13,134] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:24,100] INFO : Step 26200/50000 rank0 | Train Loss: 9.0844 | Train Loss Unweighted: 2.9533 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6276 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1364 | Regression_eta: 0.0123 | Regression_pt: 0.1351 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6725 | Classification_binary: 2.6276 | Regression_cos_phi: 0.6333 | Regression_energy: 1.8368 | Regression_eta: 0.5151 | Regression_pt: 1.9752 | Regression_sin_phi: 0.8240 | LR: 5.16e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0516 | max_loss: 0.0032 | max_backward: 0.0341 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3652 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4789.6 | uss_MB: 4309.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:29:24,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:35,273] INFO : Step 26300/50000 rank0 | Train Loss: 8.3810 | Train Loss Unweighted: 2.8987 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6005 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1239 | Regression_eta: 0.0098 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7266 | Classification_binary: 2.6005 | Regression_cos_phi: 0.5110 | Regression_energy: 1.6684 | Regression_eta: 0.4100 | Regression_pt: 1.8395 | Regression_sin_phi: 0.6252 | LR: 5.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0596 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 4789.6 | uss_MB: 4310.0 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:29:35,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:46,315] INFO : Step 26400/50000 rank0 | Train Loss: 8.8001 | Train Loss Unweighted: 2.9750 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6523 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1383 | Regression_eta: 0.0094 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7000 | Classification_binary: 2.6523 | Regression_cos_phi: 0.4713 | Regression_energy: 1.8621 | Regression_eta: 0.3940 | Regression_pt: 1.9974 | Regression_sin_phi: 0.7230 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0171 | max_loss: 0.0475 | max_backward: 0.0385 | max_optimizer_step: 0.1223 | max_tensorboard_logging: 0.3656 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4789.6 | uss_MB: 4312.0 | child_rss_MB: 3201.4 | allocated_MB: 210.7 | reserved_MB: 2748.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:29:46,532] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:57,864] INFO : Step 26500/50000 rank0 | Train Loss: 8.7020 | Train Loss Unweighted: 2.8341 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5011 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1444 | Regression_eta: 0.0100 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6866 | Classification_binary: 2.5011 | Regression_cos_phi: 0.4269 | Regression_energy: 1.9444 | Regression_eta: 0.4199 | Regression_pt: 2.0813 | Regression_sin_phi: 0.6418 | LR: 5.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3550s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.2302 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 4789.6 | uss_MB: 4310.0 | child_rss_MB: 3200.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:29:58,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:09,041] INFO : Step 26600/50000 rank0 | Train Loss: 9.3045 | Train Loss Unweighted: 3.0020 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6591 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1454 | Regression_eta: 0.0136 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7117 | Classification_binary: 2.6591 | Regression_cos_phi: 0.5406 | Regression_energy: 1.9581 | Regression_eta: 0.5701 | Regression_pt: 2.0930 | Regression_sin_phi: 0.7720 | LR: 5.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3492s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.3068 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4664.1 | uss_MB: 4183.5 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:30:09,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:20,302] INFO : Step 26700/50000 rank0 | Train Loss: 8.9748 | Train Loss Unweighted: 2.9701 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6266 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1486 | Regression_eta: 0.0095 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6891 | Classification_binary: 2.6266 | Regression_cos_phi: 0.4680 | Regression_energy: 2.0010 | Regression_eta: 0.3965 | Regression_pt: 2.1744 | Regression_sin_phi: 0.6191 | LR: 5.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0030 | max_backward: 0.1399 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.2918 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4664.1 | uss_MB: 4184.5 | child_rss_MB: 3225.2 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:20,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:31,365] INFO : Step 26800/50000 rank0 | Train Loss: 9.3820 | Train Loss Unweighted: 3.0534 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.7114 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1429 | Regression_eta: 0.0112 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8169 | Classification_binary: 2.7114 | Regression_cos_phi: 0.5681 | Regression_energy: 1.9235 | Regression_eta: 0.4694 | Regression_pt: 2.0965 | Regression_sin_phi: 0.7961 | LR: 4.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3389s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.0325 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.2885 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4664.1 | uss_MB: 4184.9 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:31,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:42,516] INFO : Step 26900/50000 rank0 | Train Loss: 9.4787 | Train Loss Unweighted: 3.0190 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6638 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1520 | Regression_eta: 0.0129 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7326 | Classification_binary: 2.6638 | Regression_cos_phi: 0.6039 | Regression_energy: 2.0463 | Regression_eta: 0.5407 | Regression_pt: 2.1719 | Regression_sin_phi: 0.7194 | LR: 4.96e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1651 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4664.1 | uss_MB: 4185.0 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:30:42,748] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:53,401] INFO : Step 27000/50000 rank0 | Train Loss: 9.7092 | Train Loss Unweighted: 3.1117 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7499 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1539 | Regression_eta: 0.0149 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7897 | Classification_binary: 2.7499 | Regression_cos_phi: 0.5597 | Regression_energy: 2.0725 | Regression_eta: 0.6231 | Regression_pt: 2.2000 | Regression_sin_phi: 0.7143 | LR: 4.94e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3011s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0579 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4664.1 | uss_MB: 4185.0 | child_rss_MB: 3198.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:53,618] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:53,663] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 12:30:53,744] INFO : Running validation on rank0 +rank0 [2026-10-28 12:30:53,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:30:53,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:30:54,830] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 12:30:54,976] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:30:55,107] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:30:55,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:30:55,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:30:55,320] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:30:55,390] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:30:55,460] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:30:55,573] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:30:55,698] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:30:55,820] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:30:55,893] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:30:55,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:30:56,034] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:30:56,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:30:56,172] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:30:56,288] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:30:56,294] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:30:56,297] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:30:56,300] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:30:56,301] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:30:56,303] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:30:56,306] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:30:56,307] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:30:56,310] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:30:56,313] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:30:56,314] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:30:56,316] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:30:56,319] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:30:56,322] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:30:56,322] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:30:56,325] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:31:02,359] INFO : VALIDATION | Step=27000/50000 | Train Loss=9.7092 | Valid Loss=8.8099 | Train Loss Unweighted=3.1117 | Valid Loss Unweighted=2.9414 | Stale=0 | ETA=66.9m +rank0 [2026-10-28 12:31:02,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:02,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:31:02,442] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:02,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:04,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:04,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:05,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:05,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:05,827] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:05,831] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:05,888] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:31:05,891] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:05,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:07,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:08,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:08,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:09,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:09,272] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:09,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:09,329] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:31:09,332] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:31:09,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:10,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:11,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:12,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:12,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:12,670] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:12,677] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:12,927] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:13,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:14,828] INFO : Plotted jets +rank0 [2026-10-28 12:31:15,936] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:15,969] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:16,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:16,575] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:18,449] INFO : Plotted jets +rank0 [2026-10-28 12:31:19,571] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:19,605] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 12:31:19,806] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 12:31:20,222] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:21,620] INFO : Plotted jets +rank0 [2026-10-28 12:31:22,725] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:33,905] INFO : Step 27100/50000 rank0 | Train Loss: 8.8459 | Train Loss Unweighted: 2.8971 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5642 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1422 | Regression_eta: 0.0093 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7237 | Classification_binary: 2.5642 | Regression_cos_phi: 0.4974 | Regression_energy: 1.9144 | Regression_eta: 0.3894 | Regression_pt: 2.0829 | Regression_sin_phi: 0.6740 | LR: 4.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4299s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1267 | max_loss: 0.0034 | max_backward: 0.0386 | max_optimizer_step: 0.1206 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4857.2 | uss_MB: 4381.4 | child_rss_MB: 3200.8 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:31:34,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:45,031] INFO : Step 27200/50000 rank0 | Train Loss: 8.7159 | Train Loss Unweighted: 2.8628 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5394 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1378 | Regression_eta: 0.0091 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7149 | Classification_binary: 2.5394 | Regression_cos_phi: 0.4609 | Regression_energy: 1.8552 | Regression_eta: 0.3809 | Regression_pt: 2.0101 | Regression_sin_phi: 0.7546 | LR: 4.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0413 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3712 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4857.2 | uss_MB: 4379.3 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:31:45,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:56,166] INFO : Step 27300/50000 rank0 | Train Loss: 8.4225 | Train Loss Unweighted: 2.8975 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5916 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1309 | Regression_eta: 0.0098 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.5916 | Regression_cos_phi: 0.4176 | Regression_energy: 1.7628 | Regression_eta: 0.4127 | Regression_pt: 1.8820 | Regression_sin_phi: 0.6641 | LR: 4.85e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3797s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0512 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:31:56,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:07,283] INFO : Step 27400/50000 rank0 | Train Loss: 8.7941 | Train Loss Unweighted: 2.9234 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6138 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1281 | Regression_eta: 0.0115 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7778 | Classification_binary: 2.6138 | Regression_cos_phi: 0.5622 | Regression_energy: 1.7241 | Regression_eta: 0.4807 | Regression_pt: 1.8557 | Regression_sin_phi: 0.7798 | LR: 4.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2955s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0576 | max_forward: 0.0157 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1242 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:07,497] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:18,272] INFO : Step 27500/50000 rank0 | Train Loss: 8.4812 | Train Loss Unweighted: 2.8603 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5534 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1301 | Regression_eta: 0.0083 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7229 | Classification_binary: 2.5534 | Regression_cos_phi: 0.4173 | Regression_energy: 1.7509 | Regression_eta: 0.3472 | Regression_pt: 1.8953 | Regression_sin_phi: 0.7942 | LR: 4.80e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0406 | max_backward: 0.0333 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:18,484] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:29,396] INFO : Step 27600/50000 rank0 | Train Loss: 9.1038 | Train Loss Unweighted: 2.9487 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6253 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1341 | Regression_eta: 0.0141 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7621 | Classification_binary: 2.6253 | Regression_cos_phi: 0.6196 | Regression_energy: 1.8049 | Regression_eta: 0.5919 | Regression_pt: 1.9235 | Regression_sin_phi: 0.7764 | LR: 4.77e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3896s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0563 | max_backward: 0.0324 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 4735.5 | uss_MB: 4256.6 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:29,611] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:40,592] INFO : Step 27700/50000 rank0 | Train Loss: 8.3890 | Train Loss Unweighted: 2.8754 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5651 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1306 | Regression_eta: 0.0091 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7154 | Classification_binary: 2.5651 | Regression_cos_phi: 0.4244 | Regression_energy: 1.7588 | Regression_eta: 0.3796 | Regression_pt: 1.9651 | Regression_sin_phi: 0.5806 | LR: 4.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0664 | max_loss: 0.0030 | max_backward: 0.0339 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.3739 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4735.5 | uss_MB: 4257.5 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:40,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:32:51,591] INFO : Step 27800/50000 rank0 | Train Loss: 8.7628 | Train Loss Unweighted: 2.9021 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5668 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1454 | Regression_eta: 0.0090 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6817 | Classification_binary: 2.5668 | Regression_cos_phi: 0.3922 | Regression_energy: 1.9575 | Regression_eta: 0.3767 | Regression_pt: 2.1240 | Regression_sin_phi: 0.6638 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0108 | max_backward: 0.0337 | max_optimizer_step: 0.1223 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4735.5 | uss_MB: 4257.6 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:32:51,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:02,810] INFO : Step 27900/50000 rank0 | Train Loss: 8.2349 | Train Loss Unweighted: 2.8685 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5744 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1238 | Regression_eta: 0.0102 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6050 | Classification_binary: 2.5744 | Regression_cos_phi: 0.4691 | Regression_energy: 1.6668 | Regression_eta: 0.4283 | Regression_pt: 1.8336 | Regression_sin_phi: 0.6578 | LR: 4.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3417s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0431 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4735.5 | uss_MB: 4257.7 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:33:03,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:13,919] INFO : Step 28000/50000 rank0 | Train Loss: 8.6612 | Train Loss Unweighted: 2.8514 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5285 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1360 | Regression_eta: 0.0109 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7079 | Classification_binary: 2.5285 | Regression_cos_phi: 0.4541 | Regression_energy: 1.8315 | Regression_eta: 0.4568 | Regression_pt: 2.0243 | Regression_sin_phi: 0.6581 | LR: 4.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1196 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.3708 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4735.5 | uss_MB: 4257.7 | child_rss_MB: 3200.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:14,133] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:14,222] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 12:33:14,304] INFO : Running validation on rank0 +rank0 [2026-10-28 12:33:14,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:33:14,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:33:15,389] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 12:33:15,533] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:33:15,665] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:33:15,739] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:33:15,809] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:33:15,876] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:33:15,946] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:33:16,015] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:33:16,127] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:33:16,252] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:33:16,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:33:16,446] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:33:16,517] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:33:16,585] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:33:16,654] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:33:16,723] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:33:16,840] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:33:16,847] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:33:16,849] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:33:16,852] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:33:16,853] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:33:16,856] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:33:16,858] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:33:16,861] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:33:16,862] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:33:16,865] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:33:16,867] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:33:16,868] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:33:16,871] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:33:16,872] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:33:16,874] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:33:16,877] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:33:22,922] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.6612 | Valid Loss=8.7574 | Train Loss Unweighted=2.8514 | Valid Loss Unweighted=2.9339 | Stale=0 | ETA=63.5m +rank0 [2026-10-28 12:33:22,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:22,993] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:33:22,997] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:23,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:24,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:25,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:25,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:26,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:26,462] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:26,466] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:26,532] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:33:26,535] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:26,630] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:27,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:28,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:29,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:29,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:29,892] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:29,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:29,948] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:33:29,951] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:33:30,074] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:31,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:32,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:32,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:33,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:33,286] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:33,294] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:33,552] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:34,011] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:35,471] INFO : Plotted jets +rank0 [2026-10-28 12:33:36,575] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:36,606] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:36,831] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:37,212] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:39,120] INFO : Plotted jets +rank0 [2026-10-28 12:33:40,235] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:40,267] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 12:33:40,474] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 12:33:40,890] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:42,297] INFO : Plotted jets +rank0 [2026-10-28 12:33:43,394] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:54,756] INFO : Step 28100/50000 rank0 | Train Loss: 8.2969 | Train Loss Unweighted: 2.8741 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5658 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1330 | Regression_eta: 0.0083 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7055 | Classification_binary: 2.5658 | Regression_cos_phi: 0.4156 | Regression_energy: 1.7900 | Regression_eta: 0.3489 | Regression_pt: 1.9259 | Regression_sin_phi: 0.5453 | LR: 4.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0145 | max_backward: 0.0336 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3677 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.2 | child_rss_MB: 3201.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:54,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:06,022] INFO : Step 28200/50000 rank0 | Train Loss: 8.8105 | Train Loss Unweighted: 3.0099 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6870 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1383 | Regression_eta: 0.0093 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7779 | Classification_binary: 2.6870 | Regression_cos_phi: 0.4373 | Regression_energy: 1.8617 | Regression_eta: 0.3880 | Regression_pt: 1.9886 | Regression_sin_phi: 0.6702 | LR: 4.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3911s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0948 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3686 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4932.5 | uss_MB: 4456.3 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:06,235] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:34:17,181] INFO : Step 28300/50000 rank0 | Train Loss: 8.5038 | Train Loss Unweighted: 2.9983 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6963 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1292 | Regression_eta: 0.0092 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7273 | Classification_binary: 2.6963 | Regression_cos_phi: 0.4753 | Regression_energy: 1.7395 | Regression_eta: 0.3876 | Regression_pt: 1.8300 | Regression_sin_phi: 0.6479 | LR: 4.57e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0922 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3203.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:17,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:28,324] INFO : Step 28400/50000 rank0 | Train Loss: 9.0222 | Train Loss Unweighted: 2.9260 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.5867 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1462 | Regression_eta: 0.0113 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7713 | Classification_binary: 2.5867 | Regression_cos_phi: 0.4744 | Regression_energy: 1.9679 | Regression_eta: 0.4717 | Regression_pt: 2.0738 | Regression_sin_phi: 0.6762 | LR: 4.55e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1227 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3643 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3204.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:34:28,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:39,463] INFO : Step 28500/50000 rank0 | Train Loss: 8.7565 | Train Loss Unweighted: 2.9061 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5880 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1350 | Regression_eta: 0.0090 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6799 | Classification_binary: 2.5880 | Regression_cos_phi: 0.5340 | Regression_energy: 1.8174 | Regression_eta: 0.3780 | Regression_pt: 1.9618 | Regression_sin_phi: 0.7973 | LR: 4.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1218 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3700 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:34:39,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:50,634] INFO : Step 28600/50000 rank0 | Train Loss: 8.5012 | Train Loss Unweighted: 2.8955 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5916 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1298 | Regression_eta: 0.0104 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6453 | Classification_binary: 2.5916 | Regression_cos_phi: 0.5026 | Regression_energy: 1.7476 | Regression_eta: 0.4353 | Regression_pt: 1.8458 | Regression_sin_phi: 0.7331 | LR: 4.49e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3527s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0236 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:34:50,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:01,646] INFO : Step 28700/50000 rank0 | Train Loss: 9.1998 | Train Loss Unweighted: 2.9615 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6217 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1452 | Regression_eta: 0.0119 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7488 | Classification_binary: 2.6217 | Regression_cos_phi: 0.5026 | Regression_energy: 1.9545 | Regression_eta: 0.5006 | Regression_pt: 2.0662 | Regression_sin_phi: 0.8053 | LR: 4.46e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3616s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0439 | max_backward: 0.0338 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3752 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4808.5 | uss_MB: 4332.5 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:01,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:12,602] INFO : Step 28800/50000 rank0 | Train Loss: 8.3481 | Train Loss Unweighted: 2.7553 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4499 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1284 | Regression_eta: 0.0101 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6734 | Classification_binary: 2.4499 | Regression_cos_phi: 0.4899 | Regression_energy: 1.7291 | Regression_eta: 0.4240 | Regression_pt: 1.8900 | Regression_sin_phi: 0.6917 | LR: 4.44e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2370s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0430 | max_optimizer_step: 0.1237 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4808.5 | uss_MB: 4332.5 | child_rss_MB: 3222.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:12,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:23,640] INFO : Step 28900/50000 rank0 | Train Loss: 8.4294 | Train Loss Unweighted: 2.8640 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5543 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1328 | Regression_eta: 0.0101 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.5543 | Regression_cos_phi: 0.4486 | Regression_energy: 1.7873 | Regression_eta: 0.4229 | Regression_pt: 1.9168 | Regression_sin_phi: 0.6373 | LR: 4.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4808.5 | uss_MB: 4332.6 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:23,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:34,599] INFO : Step 29000/50000 rank0 | Train Loss: 8.5618 | Train Loss Unweighted: 2.8650 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5619 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1266 | Regression_eta: 0.0107 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.5619 | Regression_cos_phi: 0.4800 | Regression_energy: 1.7046 | Regression_eta: 0.4503 | Regression_pt: 1.8342 | Regression_sin_phi: 0.8087 | LR: 4.38e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.1975s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0032 | max_backward: 0.0324 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4808.5 | uss_MB: 4332.6 | child_rss_MB: 3205.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:35:34,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:34,832] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 12:35:34,915] INFO : Running validation on rank0 +rank0 [2026-10-28 12:35:34,915] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:35:34,915] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:35:36,003] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 12:35:36,148] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:35:36,280] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:35:36,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:35:36,422] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:35:36,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:35:36,559] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:35:36,629] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:35:36,741] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:35:36,866] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:35:36,988] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:35:37,060] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:35:37,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:35:37,197] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:35:37,266] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:35:37,335] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:35:37,452] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:35:37,459] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:35:37,462] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:35:37,465] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:35:37,468] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:35:37,470] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:35:37,471] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:35:37,474] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:35:37,477] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:35:37,479] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:35:37,480] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:35:37,483] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:35:37,486] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:35:37,488] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:35:37,491] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:35:37,492] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:35:43,296] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.5618 | Valid Loss=8.7153 | Train Loss Unweighted=2.8650 | Valid Loss Unweighted=2.9113 | Stale=0 | ETA=60.2m +rank0 [2026-10-28 12:35:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:43,392] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:35:43,396] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:43,566] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:44,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:45,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:46,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:46,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:46,825] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:35:46,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:46,885] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:35:46,887] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:46,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:48,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:49,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:49,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:50,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:50,406] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:35:50,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:50,463] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:35:50,467] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:35:50,581] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:51,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:52,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:52,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:53,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:53,718] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:35:53,726] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:53,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:54,436] INFO : Plotted number of elements +rank0 [2026-10-28 12:35:55,873] INFO : Plotted jets +rank0 [2026-10-28 12:35:56,964] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:56,998] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:57,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:57,606] INFO : Plotted number of elements +rank0 [2026-10-28 12:35:59,478] INFO : Plotted jets +rank0 [2026-10-28 12:36:00,591] INFO : Plotted jet ratio +rank0 [2026-10-28 12:36:00,625] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 12:36:00,823] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 12:36:01,239] INFO : Plotted number of elements +rank0 [2026-10-28 12:36:02,636] INFO : Plotted jets +rank0 [2026-10-28 12:36:03,728] INFO : Plotted jet ratio +rank0 [2026-10-28 12:36:14,885] INFO : Step 29100/50000 rank0 | Train Loss: 8.2495 | Train Loss Unweighted: 2.8462 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5490 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1263 | Regression_eta: 0.0096 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6456 | Classification_binary: 2.5490 | Regression_cos_phi: 0.4857 | Regression_energy: 1.7004 | Regression_eta: 0.4015 | Regression_pt: 1.8330 | Regression_sin_phi: 0.6341 | LR: 4.35e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0803 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4998.0 | uss_MB: 4521.3 | child_rss_MB: 3228.4 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:36:15,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:25,892] INFO : Step 29200/50000 rank0 | Train Loss: 8.8682 | Train Loss Unweighted: 2.9314 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6038 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1401 | Regression_eta: 0.0103 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6937 | Classification_binary: 2.6038 | Regression_cos_phi: 0.5453 | Regression_energy: 1.8857 | Regression_eta: 0.4320 | Regression_pt: 2.0194 | Regression_sin_phi: 0.6884 | LR: 4.33e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2680s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1592 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:26,125] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:36,779] INFO : Step 29300/50000 rank0 | Train Loss: 8.9416 | Train Loss Unweighted: 2.9850 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6601 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1362 | Regression_eta: 0.0099 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7113 | Classification_binary: 2.6601 | Regression_cos_phi: 0.4269 | Regression_energy: 1.8331 | Regression_eta: 0.4146 | Regression_pt: 2.0357 | Regression_sin_phi: 0.8599 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2592s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0188 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.2779 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:36:36,992] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:47,896] INFO : Step 29400/50000 rank0 | Train Loss: 9.1110 | Train Loss Unweighted: 2.9279 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5864 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1477 | Regression_eta: 0.0137 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7179 | Classification_binary: 2.5864 | Regression_cos_phi: 0.5267 | Regression_energy: 1.9887 | Regression_eta: 0.5730 | Regression_pt: 2.0619 | Regression_sin_phi: 0.6563 | LR: 4.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.3667 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:36:48,109] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:59,017] INFO : Step 29500/50000 rank0 | Train Loss: 8.7958 | Train Loss Unweighted: 2.8937 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5726 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1372 | Regression_eta: 0.0102 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7116 | Classification_binary: 2.5726 | Regression_cos_phi: 0.5096 | Regression_energy: 1.8476 | Regression_eta: 0.4270 | Regression_pt: 1.9544 | Regression_sin_phi: 0.7730 | LR: 4.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3665s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0495 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3725 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:59,228] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:10,024] INFO : Step 29600/50000 rank0 | Train Loss: 8.8940 | Train Loss Unweighted: 2.9284 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6100 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1340 | Regression_eta: 0.0103 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7340 | Classification_binary: 2.6100 | Regression_cos_phi: 0.5970 | Regression_energy: 1.8039 | Regression_eta: 0.4326 | Regression_pt: 1.9213 | Regression_sin_phi: 0.7951 | LR: 4.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0507 | max_backward: 0.0330 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:10,236] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:21,090] INFO : Step 29700/50000 rank0 | Train Loss: 8.6560 | Train Loss Unweighted: 2.8423 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5228 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1354 | Regression_eta: 0.0111 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7116 | Classification_binary: 2.5228 | Regression_cos_phi: 0.4715 | Regression_energy: 1.8222 | Regression_eta: 0.4639 | Regression_pt: 1.9704 | Regression_sin_phi: 0.6935 | LR: 4.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3097s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0152 | max_backward: 0.0340 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3636 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4884.2 | uss_MB: 4407.7 | child_rss_MB: 3202.1 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:37:21,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:32,223] INFO : Step 29800/50000 rank0 | Train Loss: 8.5682 | Train Loss Unweighted: 2.9332 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6171 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1369 | Regression_eta: 0.0102 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6788 | Classification_binary: 2.6171 | Regression_cos_phi: 0.4548 | Regression_energy: 1.8437 | Regression_eta: 0.4270 | Regression_pt: 1.9455 | Regression_sin_phi: 0.6013 | LR: 4.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.1006 | max_backward: 0.0340 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3711 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4884.2 | uss_MB: 4407.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:32,436] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:43,349] INFO : Step 29900/50000 rank0 | Train Loss: 8.5075 | Train Loss Unweighted: 2.9153 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6042 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1337 | Regression_eta: 0.0100 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6098 | Classification_binary: 2.6042 | Regression_cos_phi: 0.4452 | Regression_energy: 1.8006 | Regression_eta: 0.4188 | Regression_pt: 1.9389 | Regression_sin_phi: 0.6901 | LR: 4.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4102s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0888 | max_backward: 0.0338 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4884.2 | uss_MB: 4408.0 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:43,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:54,487] INFO : Step 30000/50000 rank0 | Train Loss: 8.3834 | Train Loss Unweighted: 2.8626 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5577 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1311 | Regression_eta: 0.0095 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7365 | Classification_binary: 2.5577 | Regression_cos_phi: 0.4218 | Regression_energy: 1.7654 | Regression_eta: 0.4000 | Regression_pt: 1.8502 | Regression_sin_phi: 0.6517 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0970 | max_backward: 0.0332 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4884.2 | uss_MB: 4408.2 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:54,701] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:54,718] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 12:37:54,902] INFO : Running validation on rank0 +rank0 [2026-10-28 12:37:54,902] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:37:54,902] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:37:56,024] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 12:37:56,171] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:37:56,303] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:37:56,378] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:37:56,448] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:37:56,515] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:37:56,585] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:37:56,655] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:37:56,769] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:37:56,894] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:37:57,016] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:37:57,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:37:57,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:37:57,229] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:37:57,299] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:37:57,369] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:37:57,487] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:37:57,494] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:37:57,497] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:37:57,500] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:37:57,501] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:37:57,503] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:37:57,506] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:37:57,509] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:37:57,512] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:37:57,513] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:37:57,515] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:37:57,516] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:37:57,519] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:37:57,521] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:37:57,524] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:37:57,525] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:38:03,336] INFO : VALIDATION | Step=30000/50000 | Train Loss=8.3834 | Valid Loss=8.6959 | Train Loss Unweighted=2.8626 | Valid Loss Unweighted=2.9131 | Stale=0 | ETA=57.0m +rank0 [2026-10-28 12:38:03,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:03,422] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:38:03,426] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:03,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:04,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:05,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:06,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:06,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:06,818] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:06,821] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:06,875] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:38:06,878] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:06,989] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:08,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:09,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:09,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:10,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:10,284] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:10,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:10,342] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:38:10,345] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:38:10,449] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:11,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:12,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:12,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:13,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:13,628] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:13,636] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:13,885] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:14,338] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:15,781] INFO : Plotted jets +rank0 [2026-10-28 12:38:16,877] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:16,909] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:17,133] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:17,510] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:19,414] INFO : Plotted jets +rank0 [2026-10-28 12:38:20,525] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:20,557] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 12:38:20,755] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 12:38:21,174] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:22,576] INFO : Plotted jets +rank0 [2026-10-28 12:38:23,670] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:35,074] INFO : Step 30100/50000 rank0 | Train Loss: 8.3520 | Train Loss Unweighted: 2.8168 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.5002 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1376 | Regression_eta: 0.0084 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6536 | Classification_binary: 2.5002 | Regression_cos_phi: 0.3959 | Regression_energy: 1.8523 | Regression_eta: 0.3539 | Regression_pt: 1.9954 | Regression_sin_phi: 0.6007 | LR: 4.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4125s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0985 | max_loss: 0.0035 | max_backward: 0.0343 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4954.7 | uss_MB: 4478.2 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:35,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:46,183] INFO : Step 30200/50000 rank0 | Train Loss: 8.4129 | Train Loss Unweighted: 2.9089 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.6181 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1190 | Regression_eta: 0.0114 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7579 | Classification_binary: 2.6181 | Regression_cos_phi: 0.4671 | Regression_energy: 1.6014 | Regression_eta: 0.4759 | Regression_pt: 1.7608 | Regression_sin_phi: 0.7318 | LR: 4.06e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4201s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1153 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3688 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4954.7 | uss_MB: 4479.2 | child_rss_MB: 3204.2 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:46,397] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:38:57,312] INFO : Step 30300/50000 rank0 | Train Loss: 9.2611 | Train Loss Unweighted: 3.0010 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6607 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1457 | Regression_eta: 0.0105 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7100 | Classification_binary: 2.6607 | Regression_cos_phi: 0.5717 | Regression_energy: 1.9615 | Regression_eta: 0.4420 | Regression_pt: 2.0776 | Regression_sin_phi: 0.8376 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4019s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0951 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3688 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4954.7 | uss_MB: 4479.3 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:38:57,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:39:08,420] INFO : Step 30400/50000 rank0 | Train Loss: 8.8546 | Train Loss Unweighted: 2.9369 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5946 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1495 | Regression_eta: 0.0085 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7313 | Classification_binary: 2.5946 | Regression_cos_phi: 0.3603 | Regression_energy: 2.0122 | Regression_eta: 0.3556 | Regression_pt: 2.1675 | Regression_sin_phi: 0.6331 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2811s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0588 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4954.7 | uss_MB: 4479.4 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:39:08,634] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:19,417] INFO : Step 30500/50000 rank0 | Train Loss: 8.7242 | Train Loss Unweighted: 2.9100 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5866 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1388 | Regression_eta: 0.0090 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.5866 | Regression_cos_phi: 0.5086 | Regression_energy: 1.8690 | Regression_eta: 0.3764 | Regression_pt: 2.0059 | Regression_sin_phi: 0.6860 | LR: 3.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3817s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0632 | max_backward: 0.0337 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3679 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4954.7 | uss_MB: 4479.3 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:19,629] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:30,646] INFO : Step 30600/50000 rank0 | Train Loss: 8.8363 | Train Loss Unweighted: 2.9371 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6006 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1461 | Regression_eta: 0.0092 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6946 | Classification_binary: 2.6006 | Regression_cos_phi: 0.3565 | Regression_energy: 1.9674 | Regression_eta: 0.3837 | Regression_pt: 2.1226 | Regression_sin_phi: 0.7108 | LR: 3.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0708 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3729 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4955.7 | uss_MB: 4479.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:30,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:41,657] INFO : Step 30700/50000 rank0 | Train Loss: 8.5080 | Train Loss Unweighted: 2.8687 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5570 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1327 | Regression_eta: 0.0118 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6806 | Classification_binary: 2.5570 | Regression_cos_phi: 0.4581 | Regression_energy: 1.7862 | Regression_eta: 0.4947 | Regression_pt: 1.9145 | Regression_sin_phi: 0.6169 | LR: 3.92e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3653s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0387 | max_backward: 0.0329 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4954.4 | uss_MB: 4478.3 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:41,872] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:52,558] INFO : Step 30800/50000 rank0 | Train Loss: 8.7070 | Train Loss Unweighted: 2.7677 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4453 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1370 | Regression_eta: 0.0103 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6667 | Classification_binary: 2.4453 | Regression_cos_phi: 0.4769 | Regression_energy: 1.8446 | Regression_eta: 0.4332 | Regression_pt: 1.9881 | Regression_sin_phi: 0.8523 | LR: 3.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2401s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0458 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4954.4 | uss_MB: 4479.4 | child_rss_MB: 3221.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:52,777] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:03,671] INFO : Step 30900/50000 rank0 | Train Loss: 8.6310 | Train Loss Unweighted: 2.9212 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5973 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1403 | Regression_eta: 0.0082 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6778 | Classification_binary: 2.5973 | Regression_cos_phi: 0.4455 | Regression_energy: 1.8883 | Regression_eta: 0.3451 | Regression_pt: 2.0350 | Regression_sin_phi: 0.6419 | LR: 3.87e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3426s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0032 | max_backward: 0.1393 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.2781 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4954.4 | uss_MB: 4479.5 | child_rss_MB: 3222.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:03,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:14,672] INFO : Step 31000/50000 rank0 | Train Loss: 8.2720 | Train Loss Unweighted: 2.8017 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.5101 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1218 | Regression_eta: 0.0094 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6499 | Classification_binary: 2.5101 | Regression_cos_phi: 0.5236 | Regression_energy: 1.6399 | Regression_eta: 0.3960 | Regression_pt: 1.7827 | Regression_sin_phi: 0.7698 | LR: 3.84e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4955.4 | uss_MB: 4479.8 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:40:14,897] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:14,920] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 12:40:15,000] INFO : Running validation on rank0 +rank0 [2026-10-28 12:40:15,000] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:40:15,001] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:40:16,189] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 12:40:16,336] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:40:16,469] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:40:16,544] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:40:16,613] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:40:16,681] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:40:16,750] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:40:16,819] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:40:16,932] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:40:17,057] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:40:17,180] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:40:17,252] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:40:17,321] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:40:17,390] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:40:17,459] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:40:17,528] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:40:17,642] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:40:17,649] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:40:17,651] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:40:17,653] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:40:17,655] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:40:17,656] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:40:17,659] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:40:17,662] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:40:17,664] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:40:17,665] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:40:17,668] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:40:17,669] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:40:17,671] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:40:17,674] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:40:17,676] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:40:17,677] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:40:23,473] INFO : VALIDATION | Step=31000/50000 | Train Loss=8.2720 | Valid Loss=8.6164 | Train Loss Unweighted=2.8017 | Valid Loss Unweighted=2.8884 | Stale=0 | ETA=53.8m +rank0 [2026-10-28 12:40:23,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:23,544] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:40:23,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:23,720] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:25,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:25,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:26,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:27,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:27,159] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:40:27,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:27,218] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:40:27,220] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:27,392] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:28,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:29,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:30,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:30,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:30,750] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:40:30,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:30,807] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:40:30,810] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:40:30,923] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:32,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:32,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:33,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:34,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:34,246] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:40:34,253] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:34,507] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:34,961] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:36,405] INFO : Plotted jets +rank0 [2026-10-28 12:40:37,504] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:37,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:37,757] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:38,139] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:40,043] INFO : Plotted jets +rank0 [2026-10-28 12:40:41,163] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:41,211] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 12:40:41,413] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 12:40:41,828] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:43,226] INFO : Plotted jets +rank0 [2026-10-28 12:40:44,335] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:55,629] INFO : Step 31100/50000 rank0 | Train Loss: 8.1563 | Train Loss Unweighted: 2.7358 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.4528 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1162 | Regression_eta: 0.0113 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7347 | Classification_binary: 2.4528 | Regression_cos_phi: 0.5140 | Regression_energy: 1.5649 | Regression_eta: 0.4724 | Regression_pt: 1.6835 | Regression_sin_phi: 0.7340 | LR: 3.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4021s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0176 | max_loss: 0.0611 | max_backward: 0.0386 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3830 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5027.3 | uss_MB: 4555.2 | child_rss_MB: 3197.3 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:40:55,845] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:06,747] INFO : Step 31200/50000 rank0 | Train Loss: 8.0745 | Train Loss Unweighted: 2.8105 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5270 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1184 | Regression_eta: 0.0097 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6720 | Classification_binary: 2.5270 | Regression_cos_phi: 0.4353 | Regression_energy: 1.5944 | Regression_eta: 0.4063 | Regression_pt: 1.7362 | Regression_sin_phi: 0.7033 | LR: 3.79e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3788s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0372 | max_backward: 0.0330 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5028.3 | uss_MB: 4554.1 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:41:06,962] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:17,845] INFO : Step 31300/50000 rank0 | Train Loss: 9.0935 | Train Loss Unweighted: 2.9418 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.6057 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1404 | Regression_eta: 0.0120 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7996 | Classification_binary: 2.6057 | Regression_cos_phi: 0.5264 | Regression_energy: 1.8904 | Regression_eta: 0.5035 | Regression_pt: 2.0701 | Regression_sin_phi: 0.6977 | LR: 3.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3783s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0545 | max_backward: 0.0328 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5028.3 | uss_MB: 4554.1 | child_rss_MB: 3194.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:41:18,055] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:28,729] INFO : Step 31400/50000 rank0 | Train Loss: 8.2347 | Train Loss Unweighted: 2.7348 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4126 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1428 | Regression_eta: 0.0082 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5965 | Classification_binary: 2.4126 | Regression_cos_phi: 0.3352 | Regression_energy: 1.9223 | Regression_eta: 0.3439 | Regression_pt: 2.0506 | Regression_sin_phi: 0.5737 | LR: 3.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2626s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0032 | max_backward: 0.0722 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 5028.3 | uss_MB: 4554.3 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:28,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:39,504] INFO : Step 31500/50000 rank0 | Train Loss: 8.8117 | Train Loss Unweighted: 2.9383 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6218 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1321 | Regression_eta: 0.0096 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6799 | Classification_binary: 2.6218 | Regression_cos_phi: 0.5327 | Regression_energy: 1.7780 | Regression_eta: 0.4042 | Regression_pt: 1.9671 | Regression_sin_phi: 0.8280 | LR: 3.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2541s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0357 | max_forward: 0.0191 | max_loss: 0.0032 | max_backward: 0.0389 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 5029.3 | uss_MB: 4556.8 | child_rss_MB: 3194.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:41:39,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:50,634] INFO : Step 31600/50000 rank0 | Train Loss: 9.2268 | Train Loss Unweighted: 2.9088 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5622 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1471 | Regression_eta: 0.0127 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7541 | Classification_binary: 2.5622 | Regression_cos_phi: 0.5048 | Regression_energy: 1.9802 | Regression_eta: 0.5331 | Regression_pt: 2.1303 | Regression_sin_phi: 0.7621 | LR: 3.69e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3514s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0415 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5029.3 | uss_MB: 4554.8 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:41:50,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:01,632] INFO : Step 31700/50000 rank0 | Train Loss: 8.4432 | Train Loss Unweighted: 2.7798 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4590 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1382 | Regression_eta: 0.0097 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.4590 | Regression_cos_phi: 0.3761 | Regression_energy: 1.8610 | Regression_eta: 0.4085 | Regression_pt: 2.0177 | Regression_sin_phi: 0.6509 | LR: 3.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0777 | max_backward: 0.0324 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5027.2 | uss_MB: 4553.3 | child_rss_MB: 3189.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:01,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:12,748] INFO : Step 31800/50000 rank0 | Train Loss: 8.6059 | Train Loss Unweighted: 2.8804 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5719 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1315 | Regression_eta: 0.0116 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6855 | Classification_binary: 2.5719 | Regression_cos_phi: 0.4474 | Regression_energy: 1.7705 | Regression_eta: 0.4882 | Regression_pt: 1.8570 | Regression_sin_phi: 0.7854 | LR: 3.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4278s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0745 | max_backward: 0.0338 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0203 | Memory: rss_MB: 5029.2 | uss_MB: 4554.3 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:12,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:23,824] INFO : Step 31900/50000 rank0 | Train Loss: 8.1421 | Train Loss Unweighted: 2.7852 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1308 | Regression_eta: 0.0083 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6229 | Classification_binary: 2.4828 | Regression_cos_phi: 0.3708 | Regression_energy: 1.7604 | Regression_eta: 0.3494 | Regression_pt: 1.8984 | Regression_sin_phi: 0.6573 | LR: 3.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0578 | max_backward: 0.0337 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5029.2 | uss_MB: 4554.4 | child_rss_MB: 3187.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:24,037] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:34,932] INFO : Step 32000/50000 rank0 | Train Loss: 8.6202 | Train Loss Unweighted: 2.7881 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4681 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1349 | Regression_eta: 0.0087 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.4681 | Regression_cos_phi: 0.5949 | Regression_energy: 1.8166 | Regression_eta: 0.3667 | Regression_pt: 1.9932 | Regression_sin_phi: 0.6890 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3684s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0553 | max_backward: 0.0331 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3717 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5030.2 | uss_MB: 4554.4 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:35,144] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:35,162] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 12:42:35,347] INFO : Running validation on rank0 +rank0 [2026-10-28 12:42:35,348] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:42:35,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:42:36,428] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 12:42:36,572] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:42:36,705] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:42:36,778] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:42:36,848] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:42:36,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:42:36,984] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:42:37,053] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:42:37,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:42:37,297] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:42:37,429] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:42:37,500] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:42:37,569] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:42:37,637] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:42:37,708] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:42:37,777] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:42:37,894] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:42:37,901] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:42:37,904] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:42:37,907] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:42:37,908] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:42:37,911] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:42:37,913] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:42:37,916] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:42:37,917] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:42:37,920] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:42:37,922] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:42:37,925] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:42:37,926] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:42:37,928] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:42:37,931] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:42:37,932] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:42:43,812] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.6202 | Valid Loss=8.5531 | Train Loss Unweighted=2.7881 | Valid Loss Unweighted=2.8658 | Stale=0 | ETA=50.7m +rank0 [2026-10-28 12:42:43,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:43,882] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:42:43,886] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:44,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:45,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:46,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:46,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:47,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:47,358] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:42:47,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:47,418] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:42:47,421] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:47,527] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:49,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:49,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:50,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:50,994] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:42:50,998] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:51,050] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:42:51,053] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:42:51,169] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:52,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:53,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:53,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:54,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:54,422] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:42:54,429] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:54,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:55,138] INFO : Plotted number of elements +rank0 [2026-10-28 12:42:56,585] INFO : Plotted jets +rank0 [2026-10-28 12:42:57,685] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:57,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:57,944] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:58,321] INFO : Plotted number of elements +rank0 [2026-10-28 12:43:00,214] INFO : Plotted jets +rank0 [2026-10-28 12:43:01,288] INFO : Plotted jet ratio +rank0 [2026-10-28 12:43:01,322] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 12:43:01,523] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 12:43:01,942] INFO : Plotted number of elements +rank0 [2026-10-28 12:43:03,342] INFO : Plotted jets +rank0 [2026-10-28 12:43:04,456] INFO : Plotted jet ratio +rank0 [2026-10-28 12:43:15,866] INFO : Step 32100/50000 rank0 | Train Loss: 8.1447 | Train Loss Unweighted: 2.7034 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4024 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1280 | Regression_eta: 0.0085 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6635 | Classification_binary: 2.4024 | Regression_cos_phi: 0.4751 | Regression_energy: 1.7238 | Regression_eta: 0.3549 | Regression_pt: 1.8719 | Regression_sin_phi: 0.6531 | LR: 3.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4624s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1228 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5227.3 | uss_MB: 4751.5 | child_rss_MB: 3191.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:43:16,081] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:26,941] INFO : Step 32200/50000 rank0 | Train Loss: 8.3482 | Train Loss Unweighted: 2.8947 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5831 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1343 | Regression_eta: 0.0081 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6638 | Classification_binary: 2.5831 | Regression_cos_phi: 0.4260 | Regression_energy: 1.8082 | Regression_eta: 0.3377 | Regression_pt: 1.9721 | Regression_sin_phi: 0.5574 | LR: 3.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3587s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0506 | max_loss: 0.0032 | max_backward: 0.0326 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3673 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 5105.7 | uss_MB: 4629.6 | child_rss_MB: 3188.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:27,153] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:38,081] INFO : Step 32300/50000 rank0 | Train Loss: 8.6741 | Train Loss Unweighted: 2.9191 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6067 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1325 | Regression_eta: 0.0098 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6723 | Classification_binary: 2.6067 | Regression_cos_phi: 0.4668 | Regression_energy: 1.7836 | Regression_eta: 0.4086 | Regression_pt: 1.9232 | Regression_sin_phi: 0.8128 | LR: 3.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1077 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3710 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:43:38,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:49,083] INFO : Step 32400/50000 rank0 | Train Loss: 8.0517 | Train Loss Unweighted: 2.7593 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4588 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1279 | Regression_eta: 0.0087 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6312 | Classification_binary: 2.4588 | Regression_cos_phi: 0.3688 | Regression_energy: 1.7222 | Regression_eta: 0.3658 | Regression_pt: 1.9156 | Regression_sin_phi: 0.5893 | LR: 3.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0579 | max_backward: 0.0336 | max_optimizer_step: 0.1265 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3188.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:43:49,298] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:00,302] INFO : Step 32500/50000 rank0 | Train Loss: 8.3953 | Train Loss Unweighted: 2.8386 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5259 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1337 | Regression_eta: 0.0103 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7200 | Classification_binary: 2.5259 | Regression_cos_phi: 0.3624 | Regression_energy: 1.7995 | Regression_eta: 0.4322 | Regression_pt: 1.9476 | Regression_sin_phi: 0.6078 | LR: 3.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3814s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0546 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3745 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3185.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:44:00,513] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:11,116] INFO : Step 32600/50000 rank0 | Train Loss: 8.5675 | Train Loss Unweighted: 2.8502 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5380 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1324 | Regression_eta: 0.0096 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6701 | Classification_binary: 2.5380 | Regression_cos_phi: 0.4650 | Regression_energy: 1.7824 | Regression_eta: 0.4039 | Regression_pt: 1.9305 | Regression_sin_phi: 0.7777 | LR: 3.43e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.0458 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.2775 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3214.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:44:11,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:22,439] INFO : Step 32700/50000 rank0 | Train Loss: 8.4819 | Train Loss Unweighted: 2.7991 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4813 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1364 | Regression_eta: 0.0092 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6560 | Classification_binary: 2.4813 | Regression_cos_phi: 0.4346 | Regression_energy: 1.8357 | Regression_eta: 0.3863 | Regression_pt: 1.9907 | Regression_sin_phi: 0.6973 | LR: 3.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.1094 | max_backward: 0.0331 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:44:22,651] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:33,536] INFO : Step 32800/50000 rank0 | Train Loss: 8.8493 | Train Loss Unweighted: 2.8964 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5637 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1432 | Regression_eta: 0.0095 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6790 | Classification_binary: 2.5637 | Regression_cos_phi: 0.4549 | Regression_energy: 1.9273 | Regression_eta: 0.3978 | Regression_pt: 2.0807 | Regression_sin_phi: 0.7458 | LR: 3.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3578s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0191 | max_backward: 0.0336 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5105.7 | uss_MB: 4629.8 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:33,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:44,831] INFO : Step 32900/50000 rank0 | Train Loss: 8.4216 | Train Loss Unweighted: 2.8324 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5215 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1334 | Regression_eta: 0.0106 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6373 | Classification_binary: 2.5215 | Regression_cos_phi: 0.4236 | Regression_energy: 1.7965 | Regression_eta: 0.4446 | Regression_pt: 1.9282 | Regression_sin_phi: 0.6699 | LR: 3.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4653s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0924 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1259 | max_tensorboard_logging: 0.4142 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:45,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:55,850] INFO : Step 33000/50000 rank0 | Train Loss: 8.6157 | Train Loss Unweighted: 2.8881 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5682 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1352 | Regression_eta: 0.0095 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.5682 | Regression_cos_phi: 0.5430 | Regression_energy: 1.8202 | Regression_eta: 0.3972 | Regression_pt: 2.0167 | Regression_sin_phi: 0.6004 | LR: 3.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4706s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0663 | max_backward: 0.0335 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5106.7 | uss_MB: 4629.9 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:56,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:56,163] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 12:44:56,246] INFO : Running validation on rank0 +rank0 [2026-10-28 12:44:56,247] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:44:56,247] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:44:57,336] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 12:44:57,483] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:44:57,616] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:44:57,691] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:44:57,761] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:44:57,828] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:44:57,898] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:44:57,968] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:44:58,082] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:44:58,208] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:44:58,330] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:44:58,403] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:44:58,474] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:44:58,543] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:44:58,613] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:44:58,682] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:44:58,778] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:44:58,785] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:44:58,788] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:44:58,790] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:44:58,791] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:44:58,794] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:44:58,797] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:44:58,799] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:44:58,800] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:44:58,804] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:44:58,806] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:44:58,809] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:44:58,812] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:44:58,813] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:44:58,816] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:44:58,819] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:45:04,830] INFO : VALIDATION | Step=33000/50000 | Train Loss=8.6157 | Valid Loss=8.5124 | Train Loss Unweighted=2.8881 | Valid Loss Unweighted=2.8510 | Stale=0 | ETA=47.7m +rank0 [2026-10-28 12:45:04,839] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:04,902] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:45:04,906] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:05,077] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:06,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:07,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:07,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:08,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:08,454] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:45:08,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:08,513] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:45:08,516] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:08,625] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:09,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:10,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:11,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:11,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:11,855] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:45:11,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:11,911] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:45:11,915] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:45:12,092] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:13,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:13,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:14,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:15,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:15,241] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:45:15,248] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:15,499] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:15,955] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:17,411] INFO : Plotted jets +rank0 [2026-10-28 12:45:18,507] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:18,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:18,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:19,144] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:21,083] INFO : Plotted jets +rank0 [2026-10-28 12:45:22,195] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:22,227] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 12:45:22,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 12:45:22,851] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:24,250] INFO : Plotted jets +rank0 [2026-10-28 12:45:25,341] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:36,505] INFO : Step 33100/50000 rank0 | Train Loss: 8.0103 | Train Loss Unweighted: 2.7976 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4994 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1296 | Regression_eta: 0.0078 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6668 | Classification_binary: 2.4994 | Regression_cos_phi: 0.3735 | Regression_energy: 1.7447 | Regression_eta: 0.3260 | Regression_pt: 1.8649 | Regression_sin_phi: 0.5350 | LR: 3.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4105s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0248 | max_backward: 0.0340 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5301.6 | uss_MB: 4826.8 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:45:36,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:47,709] INFO : Step 33200/50000 rank0 | Train Loss: 8.4517 | Train Loss Unweighted: 2.8167 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5055 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1327 | Regression_eta: 0.0098 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6728 | Classification_binary: 2.5055 | Regression_cos_phi: 0.3991 | Regression_energy: 1.7863 | Regression_eta: 0.4097 | Regression_pt: 1.9329 | Regression_sin_phi: 0.7453 | LR: 3.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3944s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0826 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3621 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5181.3 | uss_MB: 4704.6 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:45:47,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:58,745] INFO : Step 33300/50000 rank0 | Train Loss: 8.7764 | Train Loss Unweighted: 2.8352 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4978 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1451 | Regression_eta: 0.0088 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.4978 | Regression_cos_phi: 0.4637 | Regression_energy: 1.9537 | Regression_eta: 0.3676 | Regression_pt: 2.1397 | Regression_sin_phi: 0.6654 | LR: 3.26e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3594s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0060 | max_backward: 0.0326 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5181.3 | uss_MB: 4704.7 | child_rss_MB: 3188.2 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:45:58,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:09,981] INFO : Step 33400/50000 rank0 | Train Loss: 7.8486 | Train Loss Unweighted: 2.6863 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4072 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1178 | Regression_eta: 0.0086 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6466 | Classification_binary: 2.4072 | Regression_cos_phi: 0.4361 | Regression_energy: 1.5865 | Regression_eta: 0.3608 | Regression_pt: 1.7070 | Regression_sin_phi: 0.7045 | LR: 3.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4284s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1265 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3189.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:46:10,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:21,073] INFO : Step 33500/50000 rank0 | Train Loss: 9.3423 | Train Loss Unweighted: 2.9498 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5952 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1509 | Regression_eta: 0.0110 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7289 | Classification_binary: 2.5952 | Regression_cos_phi: 0.5781 | Regression_energy: 2.0310 | Regression_eta: 0.4598 | Regression_pt: 2.2149 | Regression_sin_phi: 0.7344 | LR: 3.21e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1210 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:46:21,288] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:31,859] INFO : Step 33600/50000 rank0 | Train Loss: 8.4350 | Train Loss Unweighted: 2.8977 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5968 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1263 | Regression_eta: 0.0103 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6837 | Classification_binary: 2.5968 | Regression_cos_phi: 0.4473 | Regression_energy: 1.6997 | Regression_eta: 0.4322 | Regression_pt: 1.8549 | Regression_sin_phi: 0.7204 | LR: 3.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2608s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.0442 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:32,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:42,991] INFO : Step 33700/50000 rank0 | Train Loss: 8.5533 | Train Loss Unweighted: 2.7539 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4250 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1418 | Regression_eta: 0.0101 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6485 | Classification_binary: 2.4250 | Regression_cos_phi: 0.3751 | Regression_energy: 1.9095 | Regression_eta: 0.4215 | Regression_pt: 2.0802 | Regression_sin_phi: 0.6935 | LR: 3.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.1351 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5181.3 | uss_MB: 4705.3 | child_rss_MB: 3220.2 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:43,205] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:54,092] INFO : Step 33800/50000 rank0 | Train Loss: 8.5593 | Train Loss Unweighted: 2.9054 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1350 | Regression_eta: 0.0087 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6742 | Classification_binary: 2.5905 | Regression_cos_phi: 0.5083 | Regression_energy: 1.8180 | Regression_eta: 0.3639 | Regression_pt: 1.9559 | Regression_sin_phi: 0.6485 | LR: 3.14e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0141 | max_loss: 0.0033 | max_backward: 0.0830 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5179.2 | uss_MB: 4704.5 | child_rss_MB: 3221.5 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:54,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:05,419] INFO : Step 33900/50000 rank0 | Train Loss: 9.2130 | Train Loss Unweighted: 2.9121 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5553 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1540 | Regression_eta: 0.0094 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7158 | Classification_binary: 2.5553 | Regression_cos_phi: 0.4338 | Regression_energy: 2.0734 | Regression_eta: 0.3927 | Regression_pt: 2.2581 | Regression_sin_phi: 0.7838 | LR: 3.11e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4832s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.1120 | max_backward: 0.0328 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5179.2 | uss_MB: 4704.9 | child_rss_MB: 3191.6 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:47:05,633] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:16,512] INFO : Step 34000/50000 rank0 | Train Loss: 8.8382 | Train Loss Unweighted: 2.8593 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5243 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1442 | Regression_eta: 0.0115 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6786 | Classification_binary: 2.5243 | Regression_cos_phi: 0.4835 | Regression_energy: 1.9418 | Regression_eta: 0.4823 | Regression_pt: 2.0784 | Regression_sin_phi: 0.6492 | LR: 3.09e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.4454s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0942 | max_backward: 0.0337 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5179.2 | uss_MB: 4705.0 | child_rss_MB: 3188.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:16,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:16,745] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 12:47:16,934] INFO : Running validation on rank0 +rank0 [2026-10-28 12:47:16,934] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:47:16,934] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:47:18,017] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 12:47:18,162] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:47:18,295] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:47:18,370] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:47:18,441] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:47:18,508] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:47:18,578] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:47:18,647] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:47:18,759] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:47:18,892] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:47:19,025] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:47:19,097] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:47:19,168] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:47:19,237] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:47:19,307] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:47:19,377] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:47:19,494] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:47:19,501] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:47:19,504] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:47:19,508] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:47:19,510] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:47:19,511] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:47:19,514] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:47:19,517] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:47:19,519] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:47:19,521] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:47:19,523] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:47:19,526] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:47:19,527] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:47:19,530] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:47:19,532] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:47:19,533] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:47:25,403] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.8382 | Valid Loss=8.5099 | Train Loss Unweighted=2.8593 | Valid Loss Unweighted=2.8440 | Stale=0 | ETA=44.6m +rank0 [2026-10-28 12:47:25,412] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:25,473] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:47:25,477] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:25,654] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:27,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:27,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:28,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:28,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:29,098] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:47:29,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:29,157] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:47:29,160] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:29,277] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:30,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:31,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:32,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:32,783] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:47:32,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:32,839] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:47:32,842] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:47:32,957] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:34,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:34,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:35,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:36,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:36,203] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:47:36,210] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:36,463] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:36,917] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:38,373] INFO : Plotted jets +rank0 [2026-10-28 12:47:39,478] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:39,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:39,744] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:40,123] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:42,027] INFO : Plotted jets +rank0 [2026-10-28 12:47:43,150] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:43,184] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 12:47:43,384] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 12:47:43,805] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:45,206] INFO : Plotted jets +rank0 [2026-10-28 12:47:46,323] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:57,577] INFO : Step 34100/50000 rank0 | Train Loss: 7.9387 | Train Loss Unweighted: 2.7878 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5141 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1132 | Regression_eta: 0.0093 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6896 | Classification_binary: 2.5141 | Regression_cos_phi: 0.4258 | Regression_energy: 1.5236 | Regression_eta: 0.3888 | Regression_pt: 1.6668 | Regression_sin_phi: 0.7301 | LR: 3.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1037 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5372.6 | uss_MB: 4899.7 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2724.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:47:57,790] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:08,706] INFO : Step 34200/50000 rank0 | Train Loss: 8.3928 | Train Loss Unweighted: 2.7651 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4633 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1270 | Regression_eta: 0.0095 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7008 | Classification_binary: 2.4633 | Regression_cos_phi: 0.5966 | Regression_energy: 1.7101 | Regression_eta: 0.4001 | Regression_pt: 1.8244 | Regression_sin_phi: 0.6974 | LR: 3.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1211 | max_loss: 0.0039 | max_backward: 0.0335 | max_optimizer_step: 0.1277 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5252.1 | uss_MB: 4778.4 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2724.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:48:08,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:19,803] INFO : Step 34300/50000 rank0 | Train Loss: 8.7918 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5171 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1334 | Regression_eta: 0.0118 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.5171 | Regression_cos_phi: 0.5253 | Regression_energy: 1.7960 | Regression_eta: 0.4947 | Regression_pt: 1.9439 | Regression_sin_phi: 0.8529 | LR: 3.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0768 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3685 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5252.1 | uss_MB: 4779.1 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:20,015] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:31,018] INFO : Step 34400/50000 rank0 | Train Loss: 8.8160 | Train Loss Unweighted: 2.8325 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.5131 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1329 | Regression_eta: 0.0134 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7799 | Classification_binary: 2.5131 | Regression_cos_phi: 0.5549 | Regression_energy: 1.7887 | Regression_eta: 0.5611 | Regression_pt: 1.9173 | Regression_sin_phi: 0.7009 | LR: 2.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0775 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.1 | uss_MB: 4779.1 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:48:31,236] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:42,178] INFO : Step 34500/50000 rank0 | Train Loss: 7.7804 | Train Loss Unweighted: 2.7222 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4452 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1170 | Regression_eta: 0.0087 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6659 | Classification_binary: 2.4452 | Regression_cos_phi: 0.3808 | Regression_energy: 1.5745 | Regression_eta: 0.3644 | Regression_pt: 1.7046 | Regression_sin_phi: 0.6451 | LR: 2.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1203 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3666 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.1 | uss_MB: 4779.5 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:48:42,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:53,383] INFO : Step 34600/50000 rank0 | Train Loss: 9.1739 | Train Loss Unweighted: 2.7992 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4352 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1588 | Regression_eta: 0.0108 | Regression_pt: 0.1582 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6283 | Classification_binary: 2.4352 | Regression_cos_phi: 0.4250 | Regression_energy: 2.1376 | Regression_eta: 0.4529 | Regression_pt: 2.3136 | Regression_sin_phi: 0.7813 | LR: 2.95e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1213 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5252.1 | uss_MB: 4779.7 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:53,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:04,475] INFO : Step 34700/50000 rank0 | Train Loss: 8.9338 | Train Loss Unweighted: 2.8133 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4779 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1441 | Regression_eta: 0.0119 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6721 | Classification_binary: 2.4779 | Regression_cos_phi: 0.5360 | Regression_energy: 1.9397 | Regression_eta: 0.4988 | Regression_pt: 2.0504 | Regression_sin_phi: 0.7589 | LR: 2.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4674s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1179 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5252.1 | uss_MB: 4779.7 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:49:04,686] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:15,499] INFO : Step 34800/50000 rank0 | Train Loss: 9.0514 | Train Loss Unweighted: 2.8334 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4785 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1548 | Regression_eta: 0.0091 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6535 | Classification_binary: 2.4785 | Regression_cos_phi: 0.4839 | Regression_energy: 2.0840 | Regression_eta: 0.3802 | Regression_pt: 2.2486 | Regression_sin_phi: 0.7227 | LR: 2.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4164s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0316 | max_backward: 0.0337 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5252.3 | uss_MB: 4778.1 | child_rss_MB: 3194.1 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:49:15,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:26,694] INFO : Step 34900/50000 rank0 | Train Loss: 8.5078 | Train Loss Unweighted: 2.8266 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5090 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1377 | Regression_eta: 0.0090 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6944 | Classification_binary: 2.5090 | Regression_cos_phi: 0.4532 | Regression_energy: 1.8541 | Regression_eta: 0.3753 | Regression_pt: 1.9559 | Regression_sin_phi: 0.6659 | LR: 2.88e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0481 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3708 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.3 | uss_MB: 4779.4 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:26,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:37,697] INFO : Step 35000/50000 rank0 | Train Loss: 8.2978 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4251 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1321 | Regression_eta: 0.0088 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6821 | Classification_binary: 2.4251 | Regression_cos_phi: 0.4411 | Regression_energy: 1.7788 | Regression_eta: 0.3681 | Regression_pt: 1.9509 | Regression_sin_phi: 0.6516 | LR: 2.85e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3963s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0288 | max_backward: 0.0338 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5252.3 | uss_MB: 4779.5 | child_rss_MB: 3192.9 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:49:37,908] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:37,926] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 12:49:38,113] INFO : Running validation on rank0 +rank0 [2026-10-28 12:49:38,114] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:49:38,114] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:49:39,194] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 12:49:39,340] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:49:39,472] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:49:39,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:49:39,618] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:49:39,685] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:49:39,755] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:49:39,825] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:49:39,937] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:49:40,061] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:49:40,185] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:49:40,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:49:40,329] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:49:40,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:49:40,466] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:49:40,536] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:49:40,653] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:49:40,661] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:49:40,663] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:49:40,664] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:49:40,667] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:49:40,670] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:49:40,673] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:49:40,674] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:49:40,677] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:49:40,680] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:49:40,681] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:49:40,683] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:49:40,686] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:49:40,689] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:49:40,690] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:49:40,693] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:49:46,471] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.2978 | Valid Loss=8.4354 | Train Loss Unweighted=2.7359 | Valid Loss Unweighted=2.8342 | Stale=0 | ETA=41.7m +rank0 [2026-10-28 12:49:46,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:46,543] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:49:46,547] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:46,722] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:48,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:48,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:49,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:49,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:50,025] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:50,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:50,086] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:49:50,090] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:50,195] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:51,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:52,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:52,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:53,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:53,499] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:53,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:53,557] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:49:53,559] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:49:53,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:55,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:55,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:56,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:56,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:56,845] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:56,852] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:57,106] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:57,562] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:59,004] INFO : Plotted jets +rank0 [2026-10-28 12:50:00,097] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:00,131] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:00,357] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:00,737] INFO : Plotted number of elements +rank0 [2026-10-28 12:50:02,124] INFO : Plotted jets +rank0 [2026-10-28 12:50:03,746] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:03,779] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 12:50:03,981] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 12:50:04,395] INFO : Plotted number of elements +rank0 [2026-10-28 12:50:05,791] INFO : Plotted jets +rank0 [2026-10-28 12:50:06,881] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:18,098] INFO : Step 35100/50000 rank0 | Train Loss: 8.4273 | Train Loss Unweighted: 2.8285 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5107 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1371 | Regression_eta: 0.0095 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6418 | Classification_binary: 2.5107 | Regression_cos_phi: 0.3800 | Regression_energy: 1.8455 | Regression_eta: 0.3965 | Regression_pt: 2.0063 | Regression_sin_phi: 0.6466 | LR: 2.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0972 | max_loss: 0.0033 | max_backward: 0.0326 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3676 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5443.8 | uss_MB: 4968.9 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:18,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:29,246] INFO : Step 35200/50000 rank0 | Train Loss: 8.8541 | Train Loss Unweighted: 2.8656 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5273 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1464 | Regression_eta: 0.0116 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6638 | Classification_binary: 2.5273 | Regression_cos_phi: 0.5097 | Regression_energy: 1.9705 | Regression_eta: 0.4859 | Regression_pt: 2.1056 | Regression_sin_phi: 0.5913 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4655s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0573 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.4173 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5327.2 | uss_MB: 4853.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:50:29,457] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:40,299] INFO : Step 35300/50000 rank0 | Train Loss: 8.1823 | Train Loss Unweighted: 2.8232 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5327 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1211 | Regression_eta: 0.0090 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7140 | Classification_binary: 2.5327 | Regression_cos_phi: 0.4623 | Regression_energy: 1.6309 | Regression_eta: 0.3752 | Regression_pt: 1.7877 | Regression_sin_phi: 0.6794 | LR: 2.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4224s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0740 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3696 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5328.2 | uss_MB: 4853.6 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:50:40,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:51,251] INFO : Step 35400/50000 rank0 | Train Loss: 8.0081 | Train Loss Unweighted: 2.7832 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5029 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1175 | Regression_eta: 0.0099 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7132 | Classification_binary: 2.5029 | Regression_cos_phi: 0.3957 | Regression_energy: 1.5813 | Regression_eta: 0.4170 | Regression_pt: 1.6918 | Regression_sin_phi: 0.7062 | LR: 2.76e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4041s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0353 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3027 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5328.2 | uss_MB: 4852.7 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:50:51,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:02,546] INFO : Step 35500/50000 rank0 | Train Loss: 8.3301 | Train Loss Unweighted: 2.8331 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5204 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1365 | Regression_eta: 0.0075 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6763 | Classification_binary: 2.5204 | Regression_cos_phi: 0.4288 | Regression_energy: 1.8372 | Regression_eta: 0.3155 | Regression_pt: 1.9502 | Regression_sin_phi: 0.6017 | LR: 2.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4533s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0791 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3198.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:02,760] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:13,655] INFO : Step 35600/50000 rank0 | Train Loss: 9.0884 | Train Loss Unweighted: 2.8962 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.5561 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1433 | Regression_eta: 0.0126 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7592 | Classification_binary: 2.5561 | Regression_cos_phi: 0.5283 | Regression_energy: 1.9293 | Regression_eta: 0.5291 | Regression_pt: 2.0978 | Regression_sin_phi: 0.6887 | LR: 2.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4748s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1238 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3684 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:13,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:24,680] INFO : Step 35700/50000 rank0 | Train Loss: 8.8943 | Train Loss Unweighted: 2.9542 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6287 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1384 | Regression_eta: 0.0106 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.6287 | Regression_cos_phi: 0.4580 | Regression_energy: 1.8637 | Regression_eta: 0.4432 | Regression_pt: 2.0073 | Regression_sin_phi: 0.7853 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0387 | max_backward: 0.0327 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:24,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:35,881] INFO : Step 35800/50000 rank0 | Train Loss: 8.1360 | Train Loss Unweighted: 2.7699 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4837 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1200 | Regression_eta: 0.0097 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6422 | Classification_binary: 2.4837 | Regression_cos_phi: 0.4477 | Regression_energy: 1.6153 | Regression_eta: 0.4073 | Regression_pt: 1.7441 | Regression_sin_phi: 0.7958 | LR: 2.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4320s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0819 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5328.3 | uss_MB: 4853.1 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:36,092] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:46,975] INFO : Step 35900/50000 rank0 | Train Loss: 8.1288 | Train Loss Unweighted: 2.7615 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4611 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1291 | Regression_eta: 0.0077 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6367 | Classification_binary: 2.4611 | Regression_cos_phi: 0.3908 | Regression_energy: 1.7386 | Regression_eta: 0.3212 | Regression_pt: 1.8848 | Regression_sin_phi: 0.6955 | LR: 2.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0180 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.3687 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5329.3 | uss_MB: 4853.7 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:51:47,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:58,073] INFO : Step 36000/50000 rank0 | Train Loss: 8.0806 | Train Loss Unweighted: 2.7230 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4226 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1285 | Regression_eta: 0.0115 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6575 | Classification_binary: 2.4226 | Regression_cos_phi: 0.3676 | Regression_energy: 1.7303 | Regression_eta: 0.4829 | Regression_pt: 1.8597 | Regression_sin_phi: 0.5601 | LR: 2.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3933s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0500 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.3564 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5329.3 | uss_MB: 4854.0 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:58,284] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:58,334] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 12:51:58,416] INFO : Running validation on rank0 +rank0 [2026-10-28 12:51:58,416] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:51:58,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:51:59,492] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 12:51:59,636] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:51:59,768] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:51:59,842] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:51:59,912] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:51:59,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:52:00,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:52:00,120] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:52:00,233] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:52:00,358] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:52:00,479] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:52:00,553] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:52:00,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:52:00,693] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:52:00,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:52:00,832] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:52:00,952] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:52:00,959] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:52:00,961] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:52:00,964] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:52:00,965] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:52:00,968] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:52:00,970] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:52:00,973] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:52:00,974] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:52:00,977] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:52:00,980] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:52:00,983] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:52:00,984] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:52:00,987] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:52:00,989] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:52:00,992] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:52:06,776] INFO : VALIDATION | Step=36000/50000 | Train Loss=8.0806 | Valid Loss=8.3691 | Train Loss Unweighted=2.7230 | Valid Loss Unweighted=2.8026 | Stale=0 | ETA=38.7m +rank0 [2026-10-28 12:52:06,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:06,846] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:52:06,850] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:07,020] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:08,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:09,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:09,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:10,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:10,693] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:52:10,696] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:10,752] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:52:10,755] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:10,862] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:12,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:12,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:13,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:13,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:14,156] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:52:14,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:14,213] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:52:14,216] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:52:14,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:15,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:16,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:16,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:17,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:17,591] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:52:17,599] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:17,847] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:18,300] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:19,743] INFO : Plotted jets +rank0 [2026-10-28 12:52:20,849] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:20,881] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:21,107] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:21,487] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:23,420] INFO : Plotted jets +rank0 [2026-10-28 12:52:24,542] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:24,575] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 12:52:24,778] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 12:52:25,198] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:26,596] INFO : Plotted jets +rank0 [2026-10-28 12:52:27,698] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:39,010] INFO : Step 36100/50000 rank0 | Train Loss: 8.2667 | Train Loss Unweighted: 2.8254 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5189 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1313 | Regression_eta: 0.0100 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6678 | Classification_binary: 2.5189 | Regression_cos_phi: 0.3708 | Regression_energy: 1.7672 | Regression_eta: 0.4190 | Regression_pt: 1.9177 | Regression_sin_phi: 0.6052 | LR: 2.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0408 | max_backward: 0.0329 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5518.9 | uss_MB: 5044.1 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:52:39,223] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:50,176] INFO : Step 36200/50000 rank0 | Train Loss: 8.6056 | Train Loss Unweighted: 2.8050 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4892 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1322 | Regression_eta: 0.0135 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6861 | Classification_binary: 2.4892 | Regression_cos_phi: 0.4970 | Regression_energy: 1.7804 | Regression_eta: 0.5679 | Regression_pt: 1.9373 | Regression_sin_phi: 0.6476 | LR: 2.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4699s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0611 | max_backward: 0.0336 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5518.9 | uss_MB: 5044.2 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:52:50,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:01,329] INFO : Step 36300/50000 rank0 | Train Loss: 8.0558 | Train Loss Unweighted: 2.8031 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5098 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1260 | Regression_eta: 0.0086 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6836 | Classification_binary: 2.5098 | Regression_cos_phi: 0.3991 | Regression_energy: 1.6962 | Regression_eta: 0.3595 | Regression_pt: 1.8088 | Regression_sin_phi: 0.5987 | LR: 2.57e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0539 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1290 | max_tensorboard_logging: 0.3512 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5405.1 | uss_MB: 4930.4 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:01,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:12,502] INFO : Step 36400/50000 rank0 | Train Loss: 8.3595 | Train Loss Unweighted: 2.8477 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5497 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1252 | Regression_eta: 0.0099 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6694 | Classification_binary: 2.5497 | Regression_cos_phi: 0.4667 | Regression_energy: 1.6859 | Regression_eta: 0.4139 | Regression_pt: 1.8322 | Regression_sin_phi: 0.7416 | LR: 2.55e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0848 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5405.1 | uss_MB: 4930.5 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:53:12,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:23,583] INFO : Step 36500/50000 rank0 | Train Loss: 7.7467 | Train Loss Unweighted: 2.7494 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4629 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1231 | Regression_eta: 0.0087 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5979 | Classification_binary: 2.4629 | Regression_cos_phi: 0.3608 | Regression_energy: 1.6578 | Regression_eta: 0.3631 | Regression_pt: 1.8180 | Regression_sin_phi: 0.4862 | LR: 2.52e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3571s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0227 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3639 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5405.1 | uss_MB: 4930.5 | child_rss_MB: 3201.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:53:23,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:34,695] INFO : Step 36600/50000 rank0 | Train Loss: 7.8198 | Train Loss Unweighted: 2.7002 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4185 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1193 | Regression_eta: 0.0092 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6493 | Classification_binary: 2.4185 | Regression_cos_phi: 0.4290 | Regression_energy: 1.6067 | Regression_eta: 0.3876 | Regression_pt: 1.7337 | Regression_sin_phi: 0.5949 | LR: 2.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4478s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0995 | max_loss: 0.0033 | max_backward: 0.0382 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3543 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5405.1 | uss_MB: 4933.4 | child_rss_MB: 3201.1 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:53:34,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:45,815] INFO : Step 36700/50000 rank0 | Train Loss: 8.2368 | Train Loss Unweighted: 2.8053 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5013 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1307 | Regression_eta: 0.0094 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6091 | Classification_binary: 2.5013 | Regression_cos_phi: 0.4199 | Regression_energy: 1.7597 | Regression_eta: 0.3958 | Regression_pt: 1.9010 | Regression_sin_phi: 0.6500 | LR: 2.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4496s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1075 | max_loss: 0.0034 | max_backward: 0.0322 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5405.1 | uss_MB: 4931.4 | child_rss_MB: 3206.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:46,029] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:56,957] INFO : Step 36800/50000 rank0 | Train Loss: 8.5373 | Train Loss Unweighted: 2.7573 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4489 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1296 | Regression_eta: 0.0090 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7133 | Classification_binary: 2.4489 | Regression_cos_phi: 0.4195 | Regression_energy: 1.7453 | Regression_eta: 0.3785 | Regression_pt: 1.8887 | Regression_sin_phi: 0.9431 | LR: 2.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4394s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0976 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3657 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5405.1 | uss_MB: 4931.4 | child_rss_MB: 3208.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:57,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:08,073] INFO : Step 36900/50000 rank0 | Train Loss: 8.1474 | Train Loss Unweighted: 2.6819 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3718 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1335 | Regression_eta: 0.0107 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6151 | Classification_binary: 2.3718 | Regression_cos_phi: 0.4911 | Regression_energy: 1.7968 | Regression_eta: 0.4479 | Regression_pt: 1.9401 | Regression_sin_phi: 0.4846 | LR: 2.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4086s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0021 | max_forward: 0.0429 | max_loss: 0.0035 | max_backward: 0.0379 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3635 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5404.9 | uss_MB: 4932.9 | child_rss_MB: 3206.8 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:54:08,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:19,153] INFO : Step 37000/50000 rank0 | Train Loss: 8.8223 | Train Loss Unweighted: 2.7968 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1495 | Regression_eta: 0.0109 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6895 | Classification_binary: 2.4517 | Regression_cos_phi: 0.4048 | Regression_energy: 2.0127 | Regression_eta: 0.4579 | Regression_pt: 2.1800 | Regression_sin_phi: 0.6257 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3899s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0661 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3651 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5404.9 | uss_MB: 4931.2 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:19,365] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:19,418] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 12:54:19,503] INFO : Running validation on rank0 +rank0 [2026-10-28 12:54:19,504] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:54:19,504] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:54:20,587] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 12:54:20,731] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:54:20,863] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:54:20,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:54:21,008] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:54:21,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:54:21,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:54:21,215] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:54:21,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:54:21,463] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:54:21,596] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:54:21,667] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:54:21,738] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:54:21,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:54:21,877] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:54:21,948] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:54:22,045] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:54:22,052] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:54:22,055] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:54:22,057] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:54:22,058] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:54:22,061] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:54:22,064] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:54:22,064] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:54:22,067] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:54:22,069] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:54:22,071] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:54:22,074] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:54:22,075] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:54:22,077] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:54:22,079] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:54:22,081] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:54:27,863] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.8223 | Valid Loss=8.3438 | Train Loss Unweighted=2.7968 | Valid Loss Unweighted=2.7914 | Stale=0 | ETA=35.8m +rank0 [2026-10-28 12:54:27,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:27,932] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:54:27,935] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:28,113] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:29,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:30,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:31,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:31,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:31,723] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:54:31,726] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:31,782] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:54:31,785] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:31,896] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:33,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:34,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:34,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:35,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:35,420] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:54:35,423] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:35,478] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:54:35,481] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:54:35,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:37,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:37,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:38,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:38,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:38,949] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:54:38,956] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:39,215] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:39,678] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:41,136] INFO : Plotted jets +rank0 [2026-10-28 12:54:42,256] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:42,297] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:42,522] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:42,913] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:44,865] INFO : Plotted jets +rank0 [2026-10-28 12:54:45,991] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:46,027] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:46,236] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:46,652] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:48,067] INFO : Plotted jets +rank0 [2026-10-28 12:54:49,186] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:00,717] INFO : Step 37100/50000 rank0 | Train Loss: 8.3708 | Train Loss Unweighted: 2.7524 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4462 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1302 | Regression_eta: 0.0103 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6132 | Classification_binary: 2.4462 | Regression_cos_phi: 0.4323 | Regression_energy: 1.7525 | Regression_eta: 0.4319 | Regression_pt: 1.8945 | Regression_sin_phi: 0.8002 | LR: 2.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0479 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3478 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5588.7 | uss_MB: 5116.7 | child_rss_MB: 3200.4 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:00,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:11,852] INFO : Step 37200/50000 rank0 | Train Loss: 8.5016 | Train Loss Unweighted: 2.8241 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5108 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1321 | Regression_eta: 0.0092 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6901 | Classification_binary: 2.5108 | Regression_cos_phi: 0.4730 | Regression_energy: 1.7790 | Regression_eta: 0.3875 | Regression_pt: 1.9615 | Regression_sin_phi: 0.6997 | LR: 2.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0235 | max_forward: 0.0140 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.2731 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5588.7 | uss_MB: 5116.7 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:12,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:22,927] INFO : Step 37300/50000 rank0 | Train Loss: 7.5675 | Train Loss Unweighted: 2.6832 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4055 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1194 | Regression_eta: 0.0074 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5900 | Classification_binary: 2.4055 | Regression_cos_phi: 0.3526 | Regression_energy: 1.6068 | Regression_eta: 0.3116 | Regression_pt: 1.7573 | Regression_sin_phi: 0.5437 | LR: 2.36e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4533s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1216 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3496 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5476.1 | uss_MB: 5003.1 | child_rss_MB: 3200.9 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:23,138] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:34,046] INFO : Step 37400/50000 rank0 | Train Loss: 8.0625 | Train Loss Unweighted: 2.6681 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.3753 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1233 | Regression_eta: 0.0093 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7278 | Classification_binary: 2.3753 | Regression_cos_phi: 0.4452 | Regression_energy: 1.6605 | Regression_eta: 0.3898 | Regression_pt: 1.7835 | Regression_sin_phi: 0.6804 | LR: 2.34e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0395 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3606 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5477.1 | uss_MB: 5003.4 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:34,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:45,164] INFO : Step 37500/50000 rank0 | Train Loss: 7.9934 | Train Loss Unweighted: 2.7050 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4172 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1216 | Regression_eta: 0.0087 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6362 | Classification_binary: 2.4172 | Regression_cos_phi: 0.4415 | Regression_energy: 1.6369 | Regression_eta: 0.3627 | Regression_pt: 1.7792 | Regression_sin_phi: 0.7197 | LR: 2.32e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4065s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0597 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3480 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5477.1 | uss_MB: 5003.4 | child_rss_MB: 3198.2 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:45,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:56,306] INFO : Step 37600/50000 rank0 | Train Loss: 8.5699 | Train Loss Unweighted: 2.7644 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4480 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1332 | Regression_eta: 0.0133 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6847 | Classification_binary: 2.4480 | Regression_cos_phi: 0.4207 | Regression_energy: 1.7927 | Regression_eta: 0.5595 | Regression_pt: 1.9444 | Regression_sin_phi: 0.7200 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0498 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5477.1 | uss_MB: 5003.7 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:56,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:07,376] INFO : Step 37700/50000 rank0 | Train Loss: 8.5468 | Train Loss Unweighted: 2.7946 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4779 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1350 | Regression_eta: 0.0079 | Regression_pt: 0.1342 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6674 | Classification_binary: 2.4779 | Regression_cos_phi: 0.6154 | Regression_energy: 1.8178 | Regression_eta: 0.3326 | Regression_pt: 1.9634 | Regression_sin_phi: 0.6723 | LR: 2.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0180 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1294 | max_tensorboard_logging: 0.3536 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5477.1 | uss_MB: 5005.8 | child_rss_MB: 3196.2 | allocated_MB: 210.7 | reserved_MB: 2754.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:56:07,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:18,498] INFO : Step 37800/50000 rank0 | Train Loss: 8.5405 | Train Loss Unweighted: 2.8825 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5696 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1327 | Regression_eta: 0.0106 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6910 | Classification_binary: 2.5696 | Regression_cos_phi: 0.4983 | Regression_energy: 1.7861 | Regression_eta: 0.4454 | Regression_pt: 1.9357 | Regression_sin_phi: 0.6144 | LR: 2.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0850 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5477.1 | uss_MB: 5003.8 | child_rss_MB: 3195.3 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:56:18,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:29,475] INFO : Step 37900/50000 rank0 | Train Loss: 8.5803 | Train Loss Unweighted: 2.6723 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3514 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1368 | Regression_eta: 0.0089 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6635 | Classification_binary: 2.3514 | Regression_cos_phi: 0.5806 | Regression_energy: 1.8414 | Regression_eta: 0.3738 | Regression_pt: 1.9741 | Regression_sin_phi: 0.7956 | LR: 2.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0103 | max_backward: 0.0333 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5475.3 | uss_MB: 5003.1 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:29,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:40,581] INFO : Step 38000/50000 rank0 | Train Loss: 8.2256 | Train Loss Unweighted: 2.6641 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3516 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1338 | Regression_eta: 0.0077 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6791 | Classification_binary: 2.3516 | Regression_cos_phi: 0.4444 | Regression_energy: 1.8018 | Regression_eta: 0.3229 | Regression_pt: 1.9667 | Regression_sin_phi: 0.6592 | LR: 2.22e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0221 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5476.3 | uss_MB: 5003.3 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:56:40,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:40,811] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 12:56:41,008] INFO : Running validation on rank0 +rank0 [2026-10-28 12:56:41,008] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:56:41,008] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:56:42,079] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 12:56:42,224] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:56:42,356] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:56:42,431] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:56:42,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:56:42,569] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:56:42,639] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:56:42,709] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:56:42,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:56:42,950] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:56:43,073] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:56:43,146] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:56:43,215] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:56:43,285] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:56:43,355] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:56:43,424] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:56:43,525] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:56:43,531] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:56:43,534] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:56:43,537] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:56:43,538] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:56:43,541] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:56:43,544] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:56:43,547] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:56:43,549] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:56:43,551] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:56:43,553] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:56:43,556] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:56:43,559] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:56:43,560] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:56:43,562] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:56:43,565] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:56:49,474] INFO : VALIDATION | Step=38000/50000 | Train Loss=8.2256 | Valid Loss=8.2885 | Train Loss Unweighted=2.6641 | Valid Loss Unweighted=2.7769 | Stale=0 | ETA=32.9m +rank0 [2026-10-28 12:56:49,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:49,545] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:56:49,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:56:49,729] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:51,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:51,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:52,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:52,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:53,183] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:56:53,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:53,242] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:56:53,245] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:53,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:54,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:55,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:55,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:56,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:56,702] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:56:56,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:56,761] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:56:56,764] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:56:56,882] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:58,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:59,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:59,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:57:00,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:57:00,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:57:00,246] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:57:00,497] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:57:00,958] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:02,411] INFO : Plotted jets +rank0 [2026-10-28 12:57:03,513] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:03,547] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:57:03,776] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:57:04,153] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:05,556] INFO : Plotted jets +rank0 [2026-10-28 12:57:07,168] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:07,201] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 12:57:07,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 12:57:07,824] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:09,228] INFO : Plotted jets +rank0 [2026-10-28 12:57:10,329] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:21,748] INFO : Step 38100/50000 rank0 | Train Loss: 8.1757 | Train Loss Unweighted: 2.7130 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3970 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1373 | Regression_eta: 0.0082 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6076 | Classification_binary: 2.3970 | Regression_cos_phi: 0.4134 | Regression_energy: 1.8480 | Regression_eta: 0.3434 | Regression_pt: 2.0191 | Regression_sin_phi: 0.5472 | LR: 2.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4477s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1029 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3519 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5668.2 | uss_MB: 5195.1 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:21,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:32,919] INFO : Step 38200/50000 rank0 | Train Loss: 8.1299 | Train Loss Unweighted: 2.6663 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3622 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1294 | Regression_eta: 0.0115 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6721 | Classification_binary: 2.3622 | Regression_cos_phi: 0.3757 | Regression_energy: 1.7418 | Regression_eta: 0.4815 | Regression_pt: 1.8830 | Regression_sin_phi: 0.6136 | LR: 2.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0826 | max_forward: 0.0161 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1339 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5668.2 | uss_MB: 5195.2 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:57:33,132] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:43,673] INFO : Step 38300/50000 rank0 | Train Loss: 8.0930 | Train Loss Unweighted: 2.7598 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4686 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1220 | Regression_eta: 0.0095 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6640 | Classification_binary: 2.4686 | Regression_cos_phi: 0.4356 | Regression_energy: 1.6426 | Regression_eta: 0.3989 | Regression_pt: 1.8068 | Regression_sin_phi: 0.6766 | LR: 2.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0637 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.3 | uss_MB: 5076.6 | child_rss_MB: 3227.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:43,885] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:54,475] INFO : Step 38400/50000 rank0 | Train Loss: 8.3381 | Train Loss Unweighted: 2.7588 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4569 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1268 | Regression_eta: 0.0096 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6557 | Classification_binary: 2.4569 | Regression_cos_phi: 0.6009 | Regression_energy: 1.7077 | Regression_eta: 0.4036 | Regression_pt: 1.8546 | Regression_sin_phi: 0.6587 | LR: 2.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3147s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0257 | max_forward: 0.0156 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.2724 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.3 | uss_MB: 5077.5 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:54,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:05,590] INFO : Step 38500/50000 rank0 | Train Loss: 7.7928 | Train Loss Unweighted: 2.7030 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4219 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1184 | Regression_eta: 0.0095 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6697 | Classification_binary: 2.4219 | Regression_cos_phi: 0.4504 | Regression_energy: 1.5942 | Regression_eta: 0.3991 | Regression_pt: 1.7333 | Regression_sin_phi: 0.5241 | LR: 2.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3491s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0821 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.2760 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5549.3 | uss_MB: 5077.6 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:58:05,803] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:16,583] INFO : Step 38600/50000 rank0 | Train Loss: 8.5878 | Train Loss Unweighted: 2.8573 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5402 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1335 | Regression_eta: 0.0109 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6260 | Classification_binary: 2.5402 | Regression_cos_phi: 0.5001 | Regression_energy: 1.7974 | Regression_eta: 0.4560 | Regression_pt: 1.9985 | Regression_sin_phi: 0.6694 | LR: 2.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4794s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.0983 | max_backward: 0.0317 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5550.3 | uss_MB: 5077.6 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:16,796] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:27,692] INFO : Step 38700/50000 rank0 | Train Loss: 8.8623 | Train Loss Unweighted: 2.7550 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4017 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1528 | Regression_eta: 0.0091 | Regression_pt: 0.1556 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6492 | Classification_binary: 2.4017 | Regression_cos_phi: 0.4975 | Regression_energy: 2.0568 | Regression_eta: 0.3818 | Regression_pt: 2.2749 | Regression_sin_phi: 0.6004 | LR: 2.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0999 | max_backward: 0.0339 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3656 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5550.3 | uss_MB: 5077.6 | child_rss_MB: 3195.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:58:27,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:38,854] INFO : Step 38800/50000 rank0 | Train Loss: 7.9375 | Train Loss Unweighted: 2.7322 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4476 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1203 | Regression_eta: 0.0084 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6337 | Classification_binary: 2.4476 | Regression_cos_phi: 0.4821 | Regression_energy: 1.6197 | Regression_eta: 0.3511 | Regression_pt: 1.7591 | Regression_sin_phi: 0.6442 | LR: 2.07e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4118s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0154 | max_loss: 0.0339 | max_backward: 0.0335 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 5550.3 | uss_MB: 5077.9 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:39,068] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:49,987] INFO : Step 38900/50000 rank0 | Train Loss: 8.5095 | Train Loss Unweighted: 2.8194 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4957 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1386 | Regression_eta: 0.0091 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6864 | Classification_binary: 2.4957 | Regression_cos_phi: 0.4073 | Regression_energy: 1.8653 | Regression_eta: 0.3828 | Regression_pt: 2.0547 | Regression_sin_phi: 0.6172 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4541s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0789 | max_backward: 0.0340 | max_optimizer_step: 0.1305 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5549.4 | uss_MB: 5076.3 | child_rss_MB: 3193.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:58:50,200] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:01,093] INFO : Step 39000/50000 rank0 | Train Loss: 7.8706 | Train Loss Unweighted: 2.6750 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3840 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1242 | Regression_eta: 0.0095 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6758 | Classification_binary: 2.3840 | Regression_cos_phi: 0.3582 | Regression_energy: 1.6719 | Regression_eta: 0.3997 | Regression_pt: 1.8063 | Regression_sin_phi: 0.5746 | LR: 2.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0668 | max_backward: 0.0336 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.4 | uss_MB: 5077.1 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:01,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:01,579] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 12:59:01,660] INFO : Running validation on rank0 +rank0 [2026-10-28 12:59:01,661] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:59:01,661] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:59:02,732] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 12:59:02,877] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:59:03,011] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:59:03,086] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:59:03,156] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:59:03,226] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:59:03,296] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:59:03,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:59:03,480] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:59:03,614] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:59:03,748] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:59:03,821] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:59:03,891] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:59:03,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:59:04,031] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:59:04,102] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:59:04,202] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:59:04,285] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:59:04,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:59:04,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:59:04,294] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:59:04,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:59:04,298] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:59:04,299] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:59:04,301] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:59:04,305] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:59:04,306] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:59:04,308] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:59:04,311] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:59:04,312] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:59:04,314] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:59:04,317] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:59:10,015] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.8706 | Valid Loss=8.2979 | Train Loss Unweighted=2.6750 | Valid Loss Unweighted=2.7831 | Stale=1 | ETA=30.1m +rank0 [2026-10-28 12:59:10,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:10,087] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:59:10,090] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:10,270] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:11,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:12,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:13,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:13,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:14,000] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:59:14,003] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:14,059] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:59:14,061] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:14,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:15,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:16,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:16,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:17,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:17,521] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:59:17,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:17,579] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:59:17,582] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:59:17,696] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:19,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:19,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:20,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:20,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:20,827] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:59:20,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:21,082] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:21,537] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:22,987] INFO : Plotted jets +rank0 [2026-10-28 12:59:24,099] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:24,132] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:24,358] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:24,738] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:26,700] INFO : Plotted jets +rank0 [2026-10-28 12:59:27,823] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:27,858] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 12:59:28,059] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 12:59:28,477] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:29,881] INFO : Plotted jets +rank0 [2026-10-28 12:59:30,987] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:42,129] INFO : Step 39100/50000 rank0 | Train Loss: 7.8538 | Train Loss Unweighted: 2.7071 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4246 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1203 | Regression_eta: 0.0094 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6516 | Classification_binary: 2.4246 | Regression_cos_phi: 0.4269 | Regression_energy: 1.6197 | Regression_eta: 0.3955 | Regression_pt: 1.7250 | Regression_sin_phi: 0.6105 | LR: 2.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0035 | max_backward: 0.1412 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.2754 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5740.2 | uss_MB: 5266.7 | child_rss_MB: 3216.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:42,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:53,498] INFO : Step 39200/50000 rank0 | Train Loss: 8.2073 | Train Loss Unweighted: 2.7348 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4369 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1249 | Regression_eta: 0.0106 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.4369 | Regression_cos_phi: 0.4206 | Regression_energy: 1.6815 | Regression_eta: 0.4450 | Regression_pt: 1.8372 | Regression_sin_phi: 0.6975 | LR: 2.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4475s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0647 | max_backward: 0.0335 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3763 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5740.2 | uss_MB: 5266.7 | child_rss_MB: 3191.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:59:53,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:04,537] INFO : Step 39300/50000 rank0 | Train Loss: 8.6417 | Train Loss Unweighted: 2.8263 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4958 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1422 | Regression_eta: 0.0104 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6156 | Classification_binary: 2.4958 | Regression_cos_phi: 0.4465 | Regression_energy: 1.9138 | Regression_eta: 0.4339 | Regression_pt: 2.1006 | Regression_sin_phi: 0.6355 | LR: 1.98e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4732s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.1067 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5624.3 | uss_MB: 5149.1 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:00:04,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:15,755] INFO : Step 39400/50000 rank0 | Train Loss: 8.6477 | Train Loss Unweighted: 2.8009 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4728 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1399 | Regression_eta: 0.0100 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5995 | Classification_binary: 2.4728 | Regression_cos_phi: 0.4430 | Regression_energy: 1.8833 | Regression_eta: 0.4171 | Regression_pt: 2.0925 | Regression_sin_phi: 0.7395 | LR: 1.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0369 | max_backward: 0.0339 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5624.3 | uss_MB: 5151.1 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:00:15,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:26,970] INFO : Step 39500/50000 rank0 | Train Loss: 8.1914 | Train Loss Unweighted: 2.7695 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4630 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1319 | Regression_eta: 0.0075 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6793 | Classification_binary: 2.4630 | Regression_cos_phi: 0.4020 | Regression_energy: 1.7761 | Regression_eta: 0.3150 | Regression_pt: 1.9254 | Regression_sin_phi: 0.6306 | LR: 1.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3816s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1066 | max_forward: 0.0166 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5625.3 | uss_MB: 5151.3 | child_rss_MB: 3190.7 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:00:27,183] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:38,042] INFO : Step 39600/50000 rank0 | Train Loss: 8.0060 | Train Loss Unweighted: 2.7391 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1244 | Regression_eta: 0.0085 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7035 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4564 | Regression_energy: 1.6751 | Regression_eta: 0.3546 | Regression_pt: 1.8428 | Regression_sin_phi: 0.5292 | LR: 1.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4284s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0849 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3539 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5625.3 | uss_MB: 5151.7 | child_rss_MB: 3191.5 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:00:38,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:49,073] INFO : Step 39700/50000 rank0 | Train Loss: 7.6408 | Train Loss Unweighted: 2.7026 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1105 | Regression_eta: 0.0103 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6782 | Classification_binary: 2.4355 | Regression_cos_phi: 0.4133 | Regression_energy: 1.4877 | Regression_eta: 0.4310 | Regression_pt: 1.6337 | Regression_sin_phi: 0.5615 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.1025 | max_backward: 0.0340 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5625.3 | uss_MB: 5151.8 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:00:49,289] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:00,188] INFO : Step 39800/50000 rank0 | Train Loss: 8.5052 | Train Loss Unweighted: 2.8328 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5177 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1337 | Regression_eta: 0.0081 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6570 | Classification_binary: 2.5177 | Regression_cos_phi: 0.5776 | Regression_energy: 1.7997 | Regression_eta: 0.3379 | Regression_pt: 1.9797 | Regression_sin_phi: 0.6356 | LR: 1.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4526s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0168 | max_loss: 0.0573 | max_backward: 0.0341 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3797 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5625.3 | uss_MB: 5151.9 | child_rss_MB: 3189.1 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:01:00,401] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:11,393] INFO : Step 39900/50000 rank0 | Train Loss: 8.2951 | Train Loss Unweighted: 2.8066 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5143 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1222 | Regression_eta: 0.0118 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6433 | Classification_binary: 2.5143 | Regression_cos_phi: 0.5178 | Regression_energy: 1.6455 | Regression_eta: 0.4925 | Regression_pt: 1.7683 | Regression_sin_phi: 0.7134 | LR: 1.88e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1051 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3438 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5624.4 | uss_MB: 5150.4 | child_rss_MB: 3190.2 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:11,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:22,504] INFO : Step 40000/50000 rank0 | Train Loss: 8.7455 | Train Loss Unweighted: 2.8509 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5195 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1413 | Regression_eta: 0.0105 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7342 | Classification_binary: 2.5195 | Regression_cos_phi: 0.4531 | Regression_energy: 1.9028 | Regression_eta: 0.4412 | Regression_pt: 2.0738 | Regression_sin_phi: 0.6209 | LR: 1.86e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0922 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3387 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5626.4 | uss_MB: 5151.3 | child_rss_MB: 3188.4 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:22,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:22,766] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 13:01:22,848] INFO : Running validation on rank0 +rank0 [2026-10-28 13:01:22,848] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:01:22,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:01:23,927] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 13:01:24,071] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:01:24,203] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:01:24,278] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:01:24,348] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:01:24,416] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:01:24,486] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:01:24,557] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:01:24,670] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:01:24,796] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:01:24,920] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:01:24,992] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:01:25,063] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:01:25,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:01:25,201] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:01:25,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:01:25,370] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:01:25,377] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:01:25,379] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:01:25,382] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:01:25,383] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:01:25,386] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:01:25,389] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:01:25,391] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:01:25,392] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:01:25,395] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:01:25,397] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:01:25,400] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:01:25,401] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:01:25,403] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:01:25,406] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:01:25,408] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:01:31,408] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.7455 | Valid Loss=8.2624 | Train Loss Unweighted=2.8509 | Valid Loss Unweighted=2.7656 | Stale=0 | ETA=27.2m +rank0 [2026-10-28 13:01:31,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:31,476] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:01:31,479] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:31,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:33,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:33,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:34,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:35,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:35,320] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:35,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:35,380] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:01:35,383] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:35,515] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:37,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:37,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:38,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:38,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:38,948] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:38,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:39,005] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:01:39,009] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:01:39,129] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:40,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:41,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:41,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:42,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:42,511] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:42,519] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:42,772] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:43,229] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:44,676] INFO : Plotted jets +rank0 [2026-10-28 13:01:45,789] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:45,823] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:46,049] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:46,429] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:47,831] INFO : Plotted jets +rank0 [2026-10-28 13:01:49,502] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:49,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:49,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:50,161] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:51,565] INFO : Plotted jets +rank0 [2026-10-28 13:01:52,669] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:04,127] INFO : Step 40100/50000 rank0 | Train Loss: 8.3315 | Train Loss Unweighted: 2.7117 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4029 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1321 | Regression_eta: 0.0101 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6561 | Classification_binary: 2.4029 | Regression_cos_phi: 0.4964 | Regression_energy: 1.7778 | Regression_eta: 0.4215 | Regression_pt: 1.8969 | Regression_sin_phi: 0.6799 | LR: 1.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3565s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0645 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0318 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2751 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5819.0 | uss_MB: 5343.7 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2664.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:04,342] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:15,160] INFO : Step 40200/50000 rank0 | Train Loss: 7.7988 | Train Loss Unweighted: 2.6511 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3757 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1146 | Regression_eta: 0.0102 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6439 | Classification_binary: 2.3757 | Regression_cos_phi: 0.4748 | Regression_energy: 1.5425 | Regression_eta: 0.4255 | Regression_pt: 1.6770 | Regression_sin_phi: 0.6594 | LR: 1.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0038 | max_backward: 0.0330 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3452 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5819.0 | uss_MB: 5343.8 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:02:15,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:26,270] INFO : Step 40300/50000 rank0 | Train Loss: 7.9616 | Train Loss Unweighted: 2.7298 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4416 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1220 | Regression_eta: 0.0091 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6653 | Classification_binary: 2.4416 | Regression_cos_phi: 0.4716 | Regression_energy: 1.6422 | Regression_eta: 0.3821 | Regression_pt: 1.7758 | Regression_sin_phi: 0.5830 | LR: 1.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3746s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0325 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3458 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5819.0 | uss_MB: 5343.8 | child_rss_MB: 3194.4 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:02:26,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:37,365] INFO : Step 40400/50000 rank0 | Train Loss: 8.1841 | Train Loss Unweighted: 2.7114 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1344 | Regression_eta: 0.0092 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6481 | Classification_binary: 2.4026 | Regression_cos_phi: 0.3488 | Regression_energy: 1.8099 | Regression_eta: 0.3874 | Regression_pt: 1.9191 | Regression_sin_phi: 0.6682 | LR: 1.79e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3781s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0451 | max_loss: 0.0036 | max_backward: 0.0384 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3373 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5699.7 | uss_MB: 5226.5 | child_rss_MB: 3193.3 | allocated_MB: 210.7 | reserved_MB: 2760.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:02:37,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:48,501] INFO : Step 40500/50000 rank0 | Train Loss: 8.6153 | Train Loss Unweighted: 2.7778 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4537 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1377 | Regression_eta: 0.0107 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7101 | Classification_binary: 2.4537 | Regression_cos_phi: 0.5199 | Regression_energy: 1.8539 | Regression_eta: 0.4491 | Regression_pt: 2.0095 | Regression_sin_phi: 0.6191 | LR: 1.78e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0347 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.2727 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5700.7 | uss_MB: 5224.8 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:48,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:59,573] INFO : Step 40600/50000 rank0 | Train Loss: 8.0934 | Train Loss Unweighted: 2.7617 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4632 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1271 | Regression_eta: 0.0082 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7013 | Classification_binary: 2.4632 | Regression_cos_phi: 0.4043 | Regression_energy: 1.7116 | Regression_eta: 0.3442 | Regression_pt: 1.8643 | Regression_sin_phi: 0.6045 | LR: 1.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0701 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3405 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5700.7 | uss_MB: 5224.8 | child_rss_MB: 3191.9 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:02:59,783] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:10,671] INFO : Step 40700/50000 rank0 | Train Loss: 7.5546 | Train Loss Unweighted: 2.6525 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3714 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1216 | Regression_eta: 0.0063 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5883 | Classification_binary: 2.3714 | Regression_cos_phi: 0.3792 | Regression_energy: 1.6371 | Regression_eta: 0.2636 | Regression_pt: 1.7878 | Regression_sin_phi: 0.5272 | LR: 1.75e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0666 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3529 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5701.7 | uss_MB: 5225.3 | child_rss_MB: 3191.8 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:10,886] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:21,763] INFO : Step 40800/50000 rank0 | Train Loss: 8.1880 | Train Loss Unweighted: 2.7539 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4685 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1194 | Regression_eta: 0.0103 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.4685 | Regression_cos_phi: 0.4788 | Regression_energy: 1.6078 | Regression_eta: 0.4314 | Regression_pt: 1.6953 | Regression_sin_phi: 0.7979 | LR: 1.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3590s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0205 | max_loss: 0.0036 | max_backward: 0.0329 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3515 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5701.7 | uss_MB: 5225.5 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:21,973] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:32,897] INFO : Step 40900/50000 rank0 | Train Loss: 8.1636 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4422 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1271 | Regression_eta: 0.0110 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6056 | Classification_binary: 2.4422 | Regression_cos_phi: 0.4703 | Regression_energy: 1.7117 | Regression_eta: 0.4625 | Regression_pt: 1.7545 | Regression_sin_phi: 0.7168 | LR: 1.72e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0418 | max_forward: 0.0159 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2738 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5701.7 | uss_MB: 5225.5 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:03:33,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:43,980] INFO : Step 41000/50000 rank0 | Train Loss: 8.3519 | Train Loss Unweighted: 2.8168 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5064 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1321 | Regression_eta: 0.0095 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6283 | Classification_binary: 2.5064 | Regression_cos_phi: 0.4300 | Regression_energy: 1.7779 | Regression_eta: 0.3984 | Regression_pt: 1.9644 | Regression_sin_phi: 0.6464 | LR: 1.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4155s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0741 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5701.0 | uss_MB: 5224.2 | child_rss_MB: 3192.8 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:44,190] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:44,239] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 13:03:44,321] INFO : Running validation on rank0 +rank0 [2026-10-28 13:03:44,321] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:03:44,321] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:03:45,405] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 13:03:45,550] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:03:45,683] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:03:45,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:03:45,828] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:03:45,896] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:03:45,966] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:03:46,037] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:03:46,151] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:03:46,276] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:03:46,399] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:03:46,472] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:03:46,542] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:03:46,610] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:03:46,681] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:03:46,751] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:03:46,851] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:03:46,857] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:03:46,860] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:03:46,863] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:03:46,863] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:03:46,866] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:03:46,869] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:03:46,870] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:03:46,872] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:03:46,875] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:03:46,876] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:03:46,878] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:03:46,881] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:03:46,882] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:03:46,884] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:03:46,887] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:03:52,899] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.3519 | Valid Loss=8.2598 | Train Loss Unweighted=2.8168 | Valid Loss Unweighted=2.7676 | Stale=0 | ETA=24.4m +rank0 [2026-10-28 13:03:52,908] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:52,968] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:03:52,971] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:03:53,156] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:54,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:55,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:55,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:56,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:56,393] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:03:56,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:56,453] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:03:56,455] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:03:56,574] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:58,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:58,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:59,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:59,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:04:00,115] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:04:00,119] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:04:00,173] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:04:00,176] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:04:00,295] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:04:01,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:04:02,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:04:02,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:04:03,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:04:03,440] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:04:03,447] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:04:03,700] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:04:04,158] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:05,608] INFO : Plotted jets +rank0 [2026-10-28 13:04:06,721] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:06,752] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:06,977] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:07,357] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:08,753] INFO : Plotted jets +rank0 [2026-10-28 13:04:10,438] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:10,470] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 13:04:10,671] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 13:04:11,089] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:12,489] INFO : Plotted jets +rank0 [2026-10-28 13:04:13,592] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:24,946] INFO : Step 41100/50000 rank0 | Train Loss: 8.7194 | Train Loss Unweighted: 2.7891 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4676 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1361 | Regression_eta: 0.0102 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6856 | Classification_binary: 2.4676 | Regression_cos_phi: 0.5980 | Regression_energy: 1.8327 | Regression_eta: 0.4270 | Regression_pt: 1.9702 | Regression_sin_phi: 0.7382 | LR: 1.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0220 | max_backward: 0.0330 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3751 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5893.2 | uss_MB: 5417.4 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2618.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:04:25,159] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:36,164] INFO : Step 41200/50000 rank0 | Train Loss: 8.5416 | Train Loss Unweighted: 2.8220 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5120 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1327 | Regression_eta: 0.0116 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6543 | Classification_binary: 2.5120 | Regression_cos_phi: 0.5100 | Regression_energy: 1.7864 | Regression_eta: 0.4866 | Regression_pt: 1.8714 | Regression_sin_phi: 0.7210 | LR: 1.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4015s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0589 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.3371 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 5893.2 | uss_MB: 5417.5 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:04:36,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:47,131] INFO : Step 41300/50000 rank0 | Train Loss: 7.3110 | Train Loss Unweighted: 2.5589 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2925 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1133 | Regression_eta: 0.0068 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6054 | Classification_binary: 2.2925 | Regression_cos_phi: 0.3877 | Regression_energy: 1.5254 | Regression_eta: 0.2836 | Regression_pt: 1.6741 | Regression_sin_phi: 0.5423 | LR: 1.66e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4380s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0187 | max_loss: 0.0036 | max_backward: 0.0391 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 5893.2 | uss_MB: 5418.3 | child_rss_MB: 3186.2 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:04:47,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:58,284] INFO : Step 41400/50000 rank0 | Train Loss: 8.4860 | Train Loss Unweighted: 2.8295 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5230 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1280 | Regression_eta: 0.0109 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7021 | Classification_binary: 2.5230 | Regression_cos_phi: 0.4525 | Regression_energy: 1.7238 | Regression_eta: 0.4569 | Regression_pt: 1.8903 | Regression_sin_phi: 0.7373 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0156 | max_backward: 0.0334 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.3641 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5775.7 | uss_MB: 5299.4 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:04:58,496] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:09,439] INFO : Step 41500/50000 rank0 | Train Loss: 8.3682 | Train Loss Unweighted: 2.7248 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.4097 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1350 | Regression_eta: 0.0097 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7432 | Classification_binary: 2.4097 | Regression_cos_phi: 0.3864 | Regression_energy: 1.8170 | Regression_eta: 0.4067 | Regression_pt: 1.9472 | Regression_sin_phi: 0.6581 | LR: 1.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4174s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0180 | max_loss: 0.0103 | max_backward: 0.0389 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5775.7 | uss_MB: 5301.9 | child_rss_MB: 3187.8 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:05:09,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:20,641] INFO : Step 41600/50000 rank0 | Train Loss: 8.6109 | Train Loss Unweighted: 2.7311 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4089 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1363 | Regression_eta: 0.0140 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6801 | Classification_binary: 2.4089 | Regression_cos_phi: 0.5013 | Regression_energy: 1.8353 | Regression_eta: 0.5888 | Regression_pt: 1.9690 | Regression_sin_phi: 0.6275 | LR: 1.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1213 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1318 | max_tensorboard_logging: 0.3426 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3186.3 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:20,851] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:31,816] INFO : Step 41700/50000 rank0 | Train Loss: 8.3995 | Train Loss Unweighted: 2.7988 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4845 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1362 | Regression_eta: 0.0079 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6028 | Classification_binary: 2.4845 | Regression_cos_phi: 0.4733 | Regression_energy: 1.8337 | Regression_eta: 0.3308 | Regression_pt: 1.9726 | Regression_sin_phi: 0.7018 | LR: 1.60e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0923 | max_forward: 0.0157 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3186.2 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:05:32,030] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:42,828] INFO : Step 41800/50000 rank0 | Train Loss: 8.4166 | Train Loss Unweighted: 2.7592 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4485 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1312 | Regression_eta: 0.0094 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7081 | Classification_binary: 2.4485 | Regression_cos_phi: 0.4951 | Regression_energy: 1.7665 | Regression_eta: 0.3947 | Regression_pt: 1.9233 | Regression_sin_phi: 0.6803 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4545s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0816 | max_backward: 0.0338 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3185.9 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:43,039] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:54,032] INFO : Step 41900/50000 rank0 | Train Loss: 8.4648 | Train Loss Unweighted: 2.7754 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4581 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1364 | Regression_eta: 0.0110 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6319 | Classification_binary: 2.4581 | Regression_cos_phi: 0.4534 | Regression_energy: 1.8369 | Regression_eta: 0.4629 | Regression_pt: 1.9686 | Regression_sin_phi: 0.6529 | LR: 1.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4580s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3323 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5776.7 | uss_MB: 5300.4 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:05:54,245] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:05,044] INFO : Step 42000/50000 rank0 | Train Loss: 8.3631 | Train Loss Unweighted: 2.8409 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5203 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1404 | Regression_eta: 0.0072 | Regression_pt: 0.1396 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6566 | Classification_binary: 2.5203 | Regression_cos_phi: 0.3605 | Regression_energy: 1.8904 | Regression_eta: 0.3015 | Regression_pt: 2.0414 | Regression_sin_phi: 0.5925 | LR: 1.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.1072 | max_backward: 0.0332 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5775.9 | uss_MB: 5299.2 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:06:05,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:05,273] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 13:06:05,457] INFO : Running validation on rank0 +rank0 [2026-10-28 13:06:05,457] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:06:05,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:06:06,544] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 13:06:06,689] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:06:06,823] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:06:06,898] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:06:06,967] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:06:07,036] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:06:07,106] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:06:07,176] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:06:07,289] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:06:07,422] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:06:07,555] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:06:07,628] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:06:07,699] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:06:07,768] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:06:07,838] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:06:07,907] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:06:08,006] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:06:08,012] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:06:08,015] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:06:08,017] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:06:08,020] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:06:08,021] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:06:08,024] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:06:08,025] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:06:08,027] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:06:08,030] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:06:08,031] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:06:08,033] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:06:08,036] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:06:08,038] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:06:08,039] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:06:08,042] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:06:14,093] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.3631 | Valid Loss=8.1967 | Train Loss Unweighted=2.8409 | Valid Loss Unweighted=2.7411 | Stale=0 | ETA=21.7m +rank0 [2026-10-28 13:06:14,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:14,162] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:06:14,165] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:14,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:15,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:16,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:17,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:17,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:17,782] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:06:17,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:17,840] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:06:17,843] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:17,965] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:19,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:19,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:20,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:21,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:21,252] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:06:21,255] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:21,310] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:06:21,313] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:06:21,433] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:22,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:23,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:23,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:24,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:24,701] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:06:24,708] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:24,959] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:25,411] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:26,863] INFO : Plotted jets +rank0 [2026-10-28 13:06:27,974] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:28,007] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:28,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:28,613] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:30,006] INFO : Plotted jets +rank0 [2026-10-28 13:06:31,640] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:31,673] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 13:06:31,873] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 13:06:32,291] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:33,693] INFO : Plotted jets +rank0 [2026-10-28 13:06:34,806] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:46,208] INFO : Step 42100/50000 rank0 | Train Loss: 8.6959 | Train Loss Unweighted: 2.7661 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4464 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1345 | Regression_eta: 0.0125 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6923 | Classification_binary: 2.4464 | Regression_cos_phi: 0.5347 | Regression_energy: 1.8104 | Regression_eta: 0.5236 | Regression_pt: 1.9458 | Regression_sin_phi: 0.7427 | LR: 1.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3746s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0594 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3199 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5967.1 | uss_MB: 5492.4 | child_rss_MB: 3193.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:06:46,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:06:57,321] INFO : Step 42200/50000 rank0 | Train Loss: 7.9474 | Train Loss Unweighted: 2.6970 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1265 | Regression_eta: 0.0092 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5913 | Classification_binary: 2.4026 | Regression_cos_phi: 0.3757 | Regression_energy: 1.7025 | Regression_eta: 0.3853 | Regression_pt: 1.8473 | Regression_sin_phi: 0.6428 | LR: 1.53e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4353s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1199 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3218 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5967.1 | uss_MB: 5492.4 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:06:57,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:06:59,186] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 13:06:59,186] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:07:08,821] INFO : Step 42300/50000 rank0 | Train Loss: 7.8425 | Train Loss Unweighted: 2.6964 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4197 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1171 | Regression_eta: 0.0115 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6194 | Classification_binary: 2.4197 | Regression_cos_phi: 0.4554 | Regression_energy: 1.5768 | Regression_eta: 0.4838 | Regression_pt: 1.6603 | Regression_sin_phi: 0.6271 | LR: 1.52e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.1181 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6006.0 | uss_MB: 5530.2 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:07:09,032] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:19,812] INFO : Step 42400/50000 rank0 | Train Loss: 7.8425 | Train Loss Unweighted: 2.7143 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4380 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1148 | Regression_eta: 0.0088 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6906 | Classification_binary: 2.4380 | Regression_cos_phi: 0.4260 | Regression_energy: 1.5460 | Regression_eta: 0.3680 | Regression_pt: 1.6962 | Regression_sin_phi: 0.6777 | LR: 1.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2950s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 5847.5 | uss_MB: 5371.7 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:07:20,040] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:31,071] INFO : Step 42500/50000 rank0 | Train Loss: 7.8462 | Train Loss Unweighted: 2.6829 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3982 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1221 | Regression_eta: 0.0088 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6587 | Classification_binary: 2.3982 | Regression_cos_phi: 0.4638 | Regression_energy: 1.6431 | Regression_eta: 0.3697 | Regression_pt: 1.7349 | Regression_sin_phi: 0.5778 | LR: 1.49e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3794s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0037 | max_backward: 0.1281 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5847.5 | uss_MB: 5372.2 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:07:31,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:42,056] INFO : Step 42600/50000 rank0 | Train Loss: 7.4513 | Train Loss Unweighted: 2.6006 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3437 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1069 | Regression_eta: 0.0086 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5946 | Classification_binary: 2.3437 | Regression_cos_phi: 0.4742 | Regression_energy: 1.4386 | Regression_eta: 0.3614 | Regression_pt: 1.5586 | Regression_sin_phi: 0.6802 | LR: 1.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3134s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5848.5 | uss_MB: 5372.5 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:42,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:53,318] INFO : Step 42700/50000 rank0 | Train Loss: 8.1799 | Train Loss Unweighted: 2.7325 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4267 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1314 | Regression_eta: 0.0098 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6217 | Classification_binary: 2.4267 | Regression_cos_phi: 0.4349 | Regression_energy: 1.7694 | Regression_eta: 0.4116 | Regression_pt: 1.9103 | Regression_sin_phi: 0.6053 | LR: 1.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2931s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0031 | max_backward: 0.0420 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2732 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5848.5 | uss_MB: 5372.7 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:07:53,530] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:04,291] INFO : Step 42800/50000 rank0 | Train Loss: 7.5789 | Train Loss Unweighted: 2.6883 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4137 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1176 | Regression_eta: 0.0083 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6291 | Classification_binary: 2.4137 | Regression_cos_phi: 0.3659 | Regression_energy: 1.5832 | Regression_eta: 0.3492 | Regression_pt: 1.7073 | Regression_sin_phi: 0.5306 | LR: 1.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3760s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5848.5 | uss_MB: 5372.7 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:04,510] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:15,160] INFO : Step 42900/50000 rank0 | Train Loss: 8.1238 | Train Loss Unweighted: 2.7166 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4233 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1243 | Regression_eta: 0.0087 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6055 | Classification_binary: 2.4233 | Regression_cos_phi: 0.4509 | Regression_energy: 1.6728 | Regression_eta: 0.3629 | Regression_pt: 1.8162 | Regression_sin_phi: 0.7922 | LR: 1.44e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3827s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0269 | max_loss: 0.0034 | max_backward: 0.0377 | max_optimizer_step: 0.1339 | max_tensorboard_logging: 0.3330 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5849.5 | uss_MB: 5375.4 | child_rss_MB: 3187.9 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:08:15,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:26,277] INFO : Step 43000/50000 rank0 | Train Loss: 8.3864 | Train Loss Unweighted: 2.7329 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4176 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1354 | Regression_eta: 0.0106 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6885 | Classification_binary: 2.4176 | Regression_cos_phi: 0.4762 | Regression_energy: 1.8234 | Regression_eta: 0.4463 | Regression_pt: 1.9408 | Regression_sin_phi: 0.5937 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0749 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5847.1 | uss_MB: 5371.3 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 13:08:26,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:26,526] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 13:08:26,608] INFO : Running validation on rank0 +rank0 [2026-10-28 13:08:26,608] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:08:26,608] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:08:27,687] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 13:08:27,831] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:08:27,962] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:08:28,037] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:08:28,106] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:08:28,173] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:08:28,242] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:08:28,312] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:08:28,425] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:08:28,559] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:08:28,690] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:08:28,762] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:08:28,832] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:08:28,900] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:08:28,970] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:08:29,039] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:08:29,138] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:08:29,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:08:29,147] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:08:29,150] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:08:29,151] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:08:29,153] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:08:29,156] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:08:29,157] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:08:29,159] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:08:29,162] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:08:29,163] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:08:29,165] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:08:29,168] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:08:29,171] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:08:29,172] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:08:29,174] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:08:35,201] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.3864 | Valid Loss=8.2576 | Train Loss Unweighted=2.7329 | Valid Loss Unweighted=2.7510 | Stale=1 | ETA=18.9m +rank0 [2026-10-28 13:08:35,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:35,271] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:08:35,274] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:35,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:37,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:37,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:38,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:38,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:39,033] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:39,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:39,094] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:08:39,097] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:39,213] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:40,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:41,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:41,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:42,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:42,638] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:42,641] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:42,695] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:08:42,698] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:08:42,821] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:44,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:44,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:45,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:45,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:46,011] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:08:46,019] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:46,269] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:46,728] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:48,174] INFO : Plotted jets +rank0 [2026-10-28 13:08:49,278] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:49,310] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:49,538] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:49,920] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:51,895] INFO : Plotted jets +rank0 [2026-10-28 13:08:52,965] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:52,996] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:53,199] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:53,618] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:55,022] INFO : Plotted jets +rank0 [2026-10-28 13:08:56,122] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:07,383] INFO : Step 43100/50000 rank0 | Train Loss: 8.3640 | Train Loss Unweighted: 2.7027 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.3862 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1340 | Regression_eta: 0.0113 | Regression_pt: 0.1352 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6996 | Classification_binary: 2.3862 | Regression_cos_phi: 0.4320 | Regression_energy: 1.8043 | Regression_eta: 0.4744 | Regression_pt: 1.9767 | Regression_sin_phi: 0.5909 | LR: 1.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3758s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0186 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3244 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:07,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:18,558] INFO : Step 43200/50000 rank0 | Train Loss: 8.4272 | Train Loss Unweighted: 2.7483 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4232 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1415 | Regression_eta: 0.0080 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6907 | Classification_binary: 2.4232 | Regression_cos_phi: 0.4351 | Regression_energy: 1.9049 | Regression_eta: 0.3352 | Regression_pt: 2.0443 | Regression_sin_phi: 0.5938 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4048s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0747 | max_forward: 0.0155 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3189.3 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:09:18,775] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:29,635] INFO : Step 43300/50000 rank0 | Train Loss: 8.0953 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3966 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1280 | Regression_eta: 0.0085 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6572 | Classification_binary: 2.3966 | Regression_cos_phi: 0.3799 | Regression_energy: 1.7231 | Regression_eta: 0.3580 | Regression_pt: 1.9229 | Regression_sin_phi: 0.6577 | LR: 1.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0961 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0340 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3187.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:29,850] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:40,724] INFO : Step 43400/50000 rank0 | Train Loss: 8.0748 | Train Loss Unweighted: 2.7226 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4308 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1233 | Regression_eta: 0.0085 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6795 | Classification_binary: 2.4308 | Regression_cos_phi: 0.4417 | Regression_energy: 1.6600 | Regression_eta: 0.3568 | Regression_pt: 1.7978 | Regression_sin_phi: 0.7083 | LR: 1.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5923.5 | uss_MB: 5447.6 | child_rss_MB: 3189.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:09:40,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:51,773] INFO : Step 43500/50000 rank0 | Train Loss: 7.6929 | Train Loss Unweighted: 2.7118 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4328 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1186 | Regression_eta: 0.0082 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6498 | Classification_binary: 2.4328 | Regression_cos_phi: 0.4191 | Regression_energy: 1.5967 | Regression_eta: 0.3448 | Regression_pt: 1.7391 | Regression_sin_phi: 0.5107 | LR: 1.37e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0409 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5923.5 | uss_MB: 5448.6 | child_rss_MB: 3185.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:09:51,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:02,908] INFO : Step 43600/50000 rank0 | Train Loss: 7.7739 | Train Loss Unweighted: 2.6227 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3469 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1143 | Regression_eta: 0.0101 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6079 | Classification_binary: 2.3469 | Regression_cos_phi: 0.5004 | Regression_energy: 1.5386 | Regression_eta: 0.4242 | Regression_pt: 1.6946 | Regression_sin_phi: 0.6612 | LR: 1.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3766s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5923.5 | uss_MB: 5448.7 | child_rss_MB: 3183.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:10:03,122] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:14,059] INFO : Step 43700/50000 rank0 | Train Loss: 8.3174 | Train Loss Unweighted: 2.7847 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4843 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1267 | Regression_eta: 0.0108 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6435 | Classification_binary: 2.4843 | Regression_cos_phi: 0.4514 | Regression_energy: 1.7051 | Regression_eta: 0.4512 | Regression_pt: 1.8501 | Regression_sin_phi: 0.7318 | LR: 1.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3770s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0637 | max_forward: 0.0164 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5923.5 | uss_MB: 5448.7 | child_rss_MB: 3185.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:14,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:25,081] INFO : Step 43800/50000 rank0 | Train Loss: 7.3242 | Train Loss Unweighted: 2.5683 | Train Loss Components: Classification: 0.0156 | Classification_binary: 2.3025 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1124 | Regression_eta: 0.0078 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5305 | Classification_binary: 2.3025 | Regression_cos_phi: 0.4403 | Regression_energy: 1.5132 | Regression_eta: 0.3259 | Regression_pt: 1.6859 | Regression_sin_phi: 0.5259 | LR: 1.34e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3573s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1361 | max_tensorboard_logging: 0.3288 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5923.5 | uss_MB: 5449.2 | child_rss_MB: 3185.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:25,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:36,249] INFO : Step 43900/50000 rank0 | Train Loss: 8.8461 | Train Loss Unweighted: 2.8260 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5003 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1383 | Regression_eta: 0.0100 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6465 | Classification_binary: 2.5003 | Regression_cos_phi: 0.5751 | Regression_energy: 1.8621 | Regression_eta: 0.4175 | Regression_pt: 2.0076 | Regression_sin_phi: 0.8370 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0558 | max_forward: 0.0169 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5923.5 | uss_MB: 5449.2 | child_rss_MB: 3185.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:36,460] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:47,361] INFO : Step 44000/50000 rank0 | Train Loss: 8.0328 | Train Loss Unweighted: 2.7417 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4506 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1224 | Regression_eta: 0.0104 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6476 | Classification_binary: 2.4506 | Regression_cos_phi: 0.4571 | Regression_energy: 1.6477 | Regression_eta: 0.4367 | Regression_pt: 1.8040 | Regression_sin_phi: 0.5891 | LR: 1.32e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3945s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0613 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1363 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5922.6 | uss_MB: 5447.8 | child_rss_MB: 3183.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:10:47,577] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:47,595] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 13:10:47,677] INFO : Running validation on rank0 +rank0 [2026-10-28 13:10:47,677] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:10:47,677] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:10:48,740] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 13:10:48,886] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:10:49,017] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:10:49,091] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:10:49,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:10:49,227] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:10:49,296] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:10:49,365] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:10:49,478] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:10:49,611] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:10:49,744] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:10:49,814] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:10:49,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:10:49,952] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:10:50,021] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:10:50,090] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:10:50,189] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:10:50,195] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:10:50,197] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:10:50,200] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:10:50,203] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:10:50,204] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:10:50,206] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:10:50,207] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:10:50,210] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:10:50,213] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:10:50,215] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:10:50,218] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:10:50,220] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:10:50,223] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:10:50,224] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:10:50,226] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:10:55,962] INFO : VALIDATION | Step=44000/50000 | Train Loss=8.0328 | Valid Loss=8.1542 | Train Loss Unweighted=2.7417 | Valid Loss Unweighted=2.7294 | Stale=0 | ETA=16.1m +rank0 [2026-10-28 13:10:55,971] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:10:56,033] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:10:56,036] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:10:56,215] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:10:57,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:10:58,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:10:58,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:10:59,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:10:59,530] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:10:59,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:10:59,590] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:10:59,593] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:10:59,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:01,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:01,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:02,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:02,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:03,002] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:11:03,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:03,059] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:11:03,062] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:11:03,181] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:04,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:05,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:05,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:06,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:06,492] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:11:06,500] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:11:06,751] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:11:07,210] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:08,650] INFO : Plotted jets +rank0 [2026-10-28 13:11:09,757] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:09,788] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:10,013] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:10,395] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:11,780] INFO : Plotted jets +rank0 [2026-10-28 13:11:13,425] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:13,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 13:11:13,660] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 13:11:14,078] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:15,481] INFO : Plotted jets +rank0 [2026-10-28 13:11:16,595] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:27,951] INFO : Step 44100/50000 rank0 | Train Loss: 8.2172 | Train Loss Unweighted: 2.7510 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4524 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1259 | Regression_eta: 0.0087 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6657 | Classification_binary: 2.4524 | Regression_cos_phi: 0.4852 | Regression_energy: 1.6955 | Regression_eta: 0.3641 | Regression_pt: 1.8499 | Regression_sin_phi: 0.7044 | LR: 1.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3714 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6112.1 | uss_MB: 5637.1 | child_rss_MB: 3181.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:28,163] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:39,146] INFO : Step 44200/50000 rank0 | Train Loss: 7.5128 | Train Loss Unweighted: 2.6238 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3650 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1088 | Regression_eta: 0.0105 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6051 | Classification_binary: 2.3650 | Regression_cos_phi: 0.4321 | Regression_energy: 1.4643 | Regression_eta: 0.4421 | Regression_pt: 1.5398 | Regression_sin_phi: 0.6644 | LR: 1.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4559s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1138 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3278 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6112.1 | uss_MB: 5637.1 | child_rss_MB: 3181.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:39,359] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:50,248] INFO : Step 44300/50000 rank0 | Train Loss: 8.6279 | Train Loss Unweighted: 2.7977 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4829 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1336 | Regression_eta: 0.0122 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6582 | Classification_binary: 2.4829 | Regression_cos_phi: 0.5281 | Regression_energy: 1.7980 | Regression_eta: 0.5097 | Regression_pt: 1.9121 | Regression_sin_phi: 0.7389 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4382s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0956 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3044 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6112.1 | uss_MB: 5637.2 | child_rss_MB: 3184.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:11:50,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:01,399] INFO : Step 44400/50000 rank0 | Train Loss: 8.0954 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4192 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1263 | Regression_eta: 0.0102 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6713 | Classification_binary: 2.4192 | Regression_cos_phi: 0.3967 | Regression_energy: 1.6997 | Regression_eta: 0.4257 | Regression_pt: 1.7697 | Regression_sin_phi: 0.7131 | LR: 1.28e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4017s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0333 | max_loss: 0.0035 | max_backward: 0.0378 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3355 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5995.1 | uss_MB: 5522.0 | child_rss_MB: 3188.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:12:01,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:12,473] INFO : Step 44500/50000 rank0 | Train Loss: 7.3138 | Train Loss Unweighted: 2.6478 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3807 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1151 | Regression_eta: 0.0074 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6076 | Classification_binary: 2.3807 | Regression_cos_phi: 0.3349 | Regression_energy: 1.5495 | Regression_eta: 0.3119 | Regression_pt: 1.6804 | Regression_sin_phi: 0.4488 | LR: 1.27e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4309s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1053 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.3112 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 5996.1 | uss_MB: 5520.9 | child_rss_MB: 3185.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:12,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:23,595] INFO : Step 44600/50000 rank0 | Train Loss: 7.7606 | Train Loss Unweighted: 2.6420 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3557 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1220 | Regression_eta: 0.0072 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6208 | Classification_binary: 2.3557 | Regression_cos_phi: 0.3942 | Regression_energy: 1.6420 | Regression_eta: 0.3007 | Regression_pt: 1.8076 | Regression_sin_phi: 0.6395 | LR: 1.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3748s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0455 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3290 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5996.1 | uss_MB: 5521.0 | child_rss_MB: 3184.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:23,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:34,790] INFO : Step 44700/50000 rank0 | Train Loss: 8.2437 | Train Loss Unweighted: 2.6901 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3871 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1284 | Regression_eta: 0.0101 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6234 | Classification_binary: 2.3871 | Regression_cos_phi: 0.5041 | Regression_energy: 1.7289 | Regression_eta: 0.4234 | Regression_pt: 1.8710 | Regression_sin_phi: 0.7059 | LR: 1.25e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0838 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5996.1 | uss_MB: 5521.4 | child_rss_MB: 3184.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:35,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:45,818] INFO : Step 44800/50000 rank0 | Train Loss: 7.7908 | Train Loss Unweighted: 2.7207 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1173 | Regression_eta: 0.0098 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6200 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4608 | Regression_energy: 1.5791 | Regression_eta: 0.4097 | Regression_pt: 1.6816 | Regression_sin_phi: 0.5952 | LR: 1.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0906 | max_loss: 0.0034 | max_backward: 0.0326 | max_optimizer_step: 0.1367 | max_tensorboard_logging: 0.3180 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5996.1 | uss_MB: 5521.4 | child_rss_MB: 3186.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:46,029] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:56,957] INFO : Step 44900/50000 rank0 | Train Loss: 8.0838 | Train Loss Unweighted: 2.7536 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4631 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1237 | Regression_eta: 0.0113 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6161 | Classification_binary: 2.4631 | Regression_cos_phi: 0.4318 | Regression_energy: 1.6656 | Regression_eta: 0.4719 | Regression_pt: 1.7703 | Regression_sin_phi: 0.6650 | LR: 1.23e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0310 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3194 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5997.1 | uss_MB: 5521.8 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:57,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:08,082] INFO : Step 45000/50000 rank0 | Train Loss: 7.9905 | Train Loss Unweighted: 2.7391 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4452 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1253 | Regression_eta: 0.0073 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5986 | Classification_binary: 2.4452 | Regression_cos_phi: 0.4390 | Regression_energy: 1.6874 | Regression_eta: 0.3046 | Regression_pt: 1.8635 | Regression_sin_phi: 0.6521 | LR: 1.22e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0918 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3298 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5997.1 | uss_MB: 5521.9 | child_rss_MB: 3185.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:13:08,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:08,350] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 13:13:08,434] INFO : Running validation on rank0 +rank0 [2026-10-28 13:13:08,434] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:13:08,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:13:09,513] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 13:13:09,658] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:13:09,791] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:13:09,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:13:09,937] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:13:10,005] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:13:10,075] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:13:10,145] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:13:10,259] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:13:10,386] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:13:10,510] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:13:10,583] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:13:10,653] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:13:10,722] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:13:10,793] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:13:10,862] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:13:10,960] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:13:10,966] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:13:10,968] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:13:10,971] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:13:10,972] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:13:10,974] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:13:10,977] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:13:10,978] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:13:10,981] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:13:10,983] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:13:10,986] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:13:10,987] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:13:10,989] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:13:10,992] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:13:10,993] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:13:10,996] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:13:16,801] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.9905 | Valid Loss=8.1417 | Train Loss Unweighted=2.7391 | Valid Loss Unweighted=2.7311 | Stale=0 | ETA=13.4m +rank0 [2026-10-28 13:13:16,810] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:16,871] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:13:16,874] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:17,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:18,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:19,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:19,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:20,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:20,848] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:13:20,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:20,908] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:13:20,911] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:21,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:22,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:23,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:23,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:24,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:24,456] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:13:24,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:24,514] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:13:24,516] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:13:24,638] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:25,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:26,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:27,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:27,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:27,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:13:28,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:28,257] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:28,719] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:30,184] INFO : Plotted jets +rank0 [2026-10-28 13:13:31,303] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:31,335] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:31,560] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:31,945] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:33,355] INFO : Plotted jets +rank0 [2026-10-28 13:13:35,045] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:35,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 13:13:35,280] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 13:13:35,703] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:37,122] INFO : Plotted jets +rank0 [2026-10-28 13:13:38,231] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:49,604] INFO : Step 45100/50000 rank0 | Train Loss: 7.8712 | Train Loss Unweighted: 2.7146 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4422 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1115 | Regression_eta: 0.0100 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6999 | Classification_binary: 2.4422 | Regression_cos_phi: 0.4933 | Regression_energy: 1.5006 | Regression_eta: 0.4190 | Regression_pt: 1.6500 | Regression_sin_phi: 0.6661 | LR: 1.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4258s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0063 | max_backward: 0.0340 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6194.6 | uss_MB: 5721.1 | child_rss_MB: 3186.1 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:13:49,818] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:00,821] INFO : Step 45200/50000 rank0 | Train Loss: 8.3033 | Train Loss Unweighted: 2.7416 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4403 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1257 | Regression_eta: 0.0102 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6565 | Classification_binary: 2.4403 | Regression_cos_phi: 0.4630 | Regression_energy: 1.6919 | Regression_eta: 0.4282 | Regression_pt: 1.8726 | Regression_sin_phi: 0.7509 | LR: 1.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0321 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.3155 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:01,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:12,042] INFO : Step 45300/50000 rank0 | Train Loss: 7.8314 | Train Loss Unweighted: 2.6900 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3971 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1267 | Regression_eta: 0.0082 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6001 | Classification_binary: 2.3971 | Regression_cos_phi: 0.3870 | Regression_energy: 1.7051 | Regression_eta: 0.3421 | Regression_pt: 1.8473 | Regression_sin_phi: 0.5527 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3948s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0697 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1396 | max_tensorboard_logging: 0.2713 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:12,257] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:23,135] INFO : Step 45400/50000 rank0 | Train Loss: 7.7189 | Train Loss Unweighted: 2.6456 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3708 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1162 | Regression_eta: 0.0111 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5721 | Classification_binary: 2.3708 | Regression_cos_phi: 0.4330 | Regression_energy: 1.5645 | Regression_eta: 0.4657 | Regression_pt: 1.6769 | Regression_sin_phi: 0.6358 | LR: 1.19e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.1087 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3103 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:23,351] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:34,247] INFO : Step 45500/50000 rank0 | Train Loss: 8.0477 | Train Loss Unweighted: 2.6347 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3545 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1160 | Regression_eta: 0.0108 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6250 | Classification_binary: 2.3545 | Regression_cos_phi: 0.5021 | Regression_energy: 1.5611 | Regression_eta: 0.4527 | Regression_pt: 1.6797 | Regression_sin_phi: 0.8727 | LR: 1.18e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3392s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.0317 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6070.6 | uss_MB: 5597.4 | child_rss_MB: 3186.9 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:34,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:45,389] INFO : Step 45600/50000 rank0 | Train Loss: 7.7775 | Train Loss Unweighted: 2.6670 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3969 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1120 | Regression_eta: 0.0122 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6476 | Classification_binary: 2.3969 | Regression_cos_phi: 0.4565 | Regression_energy: 1.5081 | Regression_eta: 0.5107 | Regression_pt: 1.6127 | Regression_sin_phi: 0.6451 | LR: 1.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4450s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1112 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3084 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6070.6 | uss_MB: 5597.9 | child_rss_MB: 3183.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:45,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:56,526] INFO : Step 45700/50000 rank0 | Train Loss: 8.4950 | Train Loss Unweighted: 2.8061 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4934 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1343 | Regression_eta: 0.0109 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6251 | Classification_binary: 2.4934 | Regression_cos_phi: 0.4800 | Regression_energy: 1.8084 | Regression_eta: 0.4587 | Regression_pt: 1.9191 | Regression_sin_phi: 0.7103 | LR: 1.16e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4092s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0714 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.3260 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6070.6 | uss_MB: 5598.2 | child_rss_MB: 3182.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:14:56,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:07,629] INFO : Step 45800/50000 rank0 | Train Loss: 7.5938 | Train Loss Unweighted: 2.6176 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3442 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1165 | Regression_eta: 0.0079 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6490 | Classification_binary: 2.3442 | Regression_cos_phi: 0.4050 | Regression_energy: 1.5682 | Regression_eta: 0.3304 | Regression_pt: 1.6752 | Regression_sin_phi: 0.6219 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0634 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3189 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6070.6 | uss_MB: 5598.0 | child_rss_MB: 3182.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:07,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:18,755] INFO : Step 45900/50000 rank0 | Train Loss: 8.6179 | Train Loss Unweighted: 2.7080 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3983 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1288 | Regression_eta: 0.0140 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6312 | Classification_binary: 2.3983 | Regression_cos_phi: 0.5843 | Regression_energy: 1.7344 | Regression_eta: 0.5850 | Regression_pt: 1.8597 | Regression_sin_phi: 0.8250 | LR: 1.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4395s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0998 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3133 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6071.6 | uss_MB: 5598.2 | child_rss_MB: 3186.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:18,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:29,867] INFO : Step 46000/50000 rank0 | Train Loss: 8.1133 | Train Loss Unweighted: 2.7475 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4581 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1218 | Regression_eta: 0.0102 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6259 | Classification_binary: 2.4581 | Regression_cos_phi: 0.4685 | Regression_energy: 1.6400 | Regression_eta: 0.4273 | Regression_pt: 1.7720 | Regression_sin_phi: 0.7213 | LR: 1.14e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1198 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3292 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 6071.6 | uss_MB: 5598.6 | child_rss_MB: 3184.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:30,079] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:30,126] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 13:15:30,208] INFO : Running validation on rank0 +rank0 [2026-10-28 13:15:30,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:15:30,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:15:31,274] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 13:15:31,421] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:15:31,553] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:15:31,627] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:15:31,695] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:15:31,763] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:15:31,833] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:15:31,902] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:15:32,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:15:32,139] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:15:32,262] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:15:32,334] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:15:32,404] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:15:32,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:15:32,541] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:15:32,611] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:15:32,709] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:15:32,716] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:15:32,718] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:15:32,721] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:15:32,724] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:15:32,725] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:15:32,728] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:15:32,730] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:15:32,731] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:15:32,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:15:32,737] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:15:32,737] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:15:32,740] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:15:32,743] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:15:32,743] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:15:32,746] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:15:38,472] INFO : VALIDATION | Step=46000/50000 | Train Loss=8.1133 | Valid Loss=8.1662 | Train Loss Unweighted=2.7475 | Valid Loss Unweighted=2.7221 | Stale=1 | ETA=10.7m +rank0 [2026-10-28 13:15:38,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:38,568] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:15:38,571] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:38,784] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:40,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:40,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:41,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:41,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:42,137] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:15:42,140] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:42,197] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:15:42,199] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:42,322] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:43,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:44,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:45,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:45,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:45,799] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:15:45,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:45,857] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:15:45,860] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:15:45,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:47,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:48,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:48,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:49,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:49,474] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:15:49,482] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:49,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:50,199] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:51,656] INFO : Plotted jets +rank0 [2026-10-28 13:15:52,774] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:52,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:53,033] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:53,523] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:54,924] INFO : Plotted jets +rank0 [2026-10-28 13:15:56,587] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:56,619] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:56,826] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:57,249] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:58,663] INFO : Plotted jets +rank0 [2026-10-28 13:15:59,782] INFO : Plotted jet ratio +rank0 [2026-10-28 13:16:11,189] INFO : Step 46100/50000 rank0 | Train Loss: 7.9889 | Train Loss Unweighted: 2.6806 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3861 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1256 | Regression_eta: 0.0089 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6019 | Classification_binary: 2.3861 | Regression_cos_phi: 0.4240 | Regression_energy: 1.6907 | Regression_eta: 0.3711 | Regression_pt: 1.8441 | Regression_sin_phi: 0.6710 | LR: 1.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4505s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0164 | max_backward: 0.0335 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6270.8 | uss_MB: 5796.7 | child_rss_MB: 3185.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:11,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:22,379] INFO : Step 46200/50000 rank0 | Train Loss: 8.3701 | Train Loss Unweighted: 2.7187 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4015 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1382 | Regression_eta: 0.0087 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6451 | Classification_binary: 2.4015 | Regression_cos_phi: 0.4969 | Regression_energy: 1.8611 | Regression_eta: 0.3652 | Regression_pt: 1.9601 | Regression_sin_phi: 0.6403 | LR: 1.13e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.4597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1056 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3073 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6270.8 | uss_MB: 5796.7 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:16:22,592] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:33,479] INFO : Step 46300/50000 rank0 | Train Loss: 8.1265 | Train Loss Unweighted: 2.6285 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3179 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1329 | Regression_eta: 0.0085 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6586 | Classification_binary: 2.3179 | Regression_cos_phi: 0.5081 | Regression_energy: 1.7890 | Regression_eta: 0.3580 | Regression_pt: 1.9537 | Regression_sin_phi: 0.5413 | LR: 1.12e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4506s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1113 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 0.3075 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6270.8 | uss_MB: 5796.8 | child_rss_MB: 3189.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 13:16:33,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:44,615] INFO : Step 46400/50000 rank0 | Train Loss: 7.6373 | Train Loss Unweighted: 2.6393 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1160 | Regression_eta: 0.0087 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6182 | Classification_binary: 2.3645 | Regression_cos_phi: 0.4263 | Regression_energy: 1.5610 | Regression_eta: 0.3652 | Regression_pt: 1.7045 | Regression_sin_phi: 0.5976 | LR: 1.11e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4639s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1053 | max_loss: 0.0034 | max_backward: 0.0387 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6270.8 | uss_MB: 5798.9 | child_rss_MB: 3189.3 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:16:44,826] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:55,623] INFO : Step 46500/50000 rank0 | Train Loss: 7.8443 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4147 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1181 | Regression_eta: 0.0084 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6795 | Classification_binary: 2.4147 | Regression_cos_phi: 0.4598 | Regression_energy: 1.5896 | Regression_eta: 0.3512 | Regression_pt: 1.6943 | Regression_sin_phi: 0.6552 | LR: 1.11e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0041 | max_backward: 0.0337 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.3664 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6144.4 | uss_MB: 5669.2 | child_rss_MB: 3190.1 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:55,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:06,829] INFO : Step 46600/50000 rank0 | Train Loss: 8.0785 | Train Loss Unweighted: 2.6678 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3713 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1259 | Regression_eta: 0.0093 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6009 | Classification_binary: 2.3713 | Regression_cos_phi: 0.4793 | Regression_energy: 1.6945 | Regression_eta: 0.3878 | Regression_pt: 1.8426 | Regression_sin_phi: 0.7023 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4232s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0801 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1396 | max_tensorboard_logging: 0.3072 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6144.4 | uss_MB: 5669.4 | child_rss_MB: 3187.0 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:17:07,043] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:17,875] INFO : Step 46700/50000 rank0 | Train Loss: 8.4293 | Train Loss Unweighted: 2.6965 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3756 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1354 | Regression_eta: 0.0100 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6669 | Classification_binary: 2.3756 | Regression_cos_phi: 0.4410 | Regression_energy: 1.8227 | Regression_eta: 0.4202 | Regression_pt: 2.0384 | Regression_sin_phi: 0.6646 | LR: 1.10e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4754s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0513 | max_backward: 0.0341 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6144.4 | uss_MB: 5669.8 | child_rss_MB: 3188.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:18,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:28,987] INFO : Step 46800/50000 rank0 | Train Loss: 8.0714 | Train Loss Unweighted: 2.6744 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3652 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1356 | Regression_eta: 0.0085 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6136 | Classification_binary: 2.3652 | Regression_cos_phi: 0.4432 | Regression_energy: 1.8257 | Regression_eta: 0.3566 | Regression_pt: 1.9327 | Regression_sin_phi: 0.5346 | LR: 1.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0172 | max_loss: 0.0034 | max_backward: 0.0376 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6145.4 | uss_MB: 5672.6 | child_rss_MB: 3189.3 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:17:29,198] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:40,148] INFO : Step 46900/50000 rank0 | Train Loss: 8.1200 | Train Loss Unweighted: 2.8019 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4978 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1324 | Regression_eta: 0.0078 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6368 | Classification_binary: 2.4978 | Regression_cos_phi: 0.4170 | Regression_energy: 1.7820 | Regression_eta: 0.3259 | Regression_pt: 1.9098 | Regression_sin_phi: 0.5507 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4746s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1167 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.3294 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6145.4 | uss_MB: 5670.5 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:40,361] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:51,355] INFO : Step 47000/50000 rank0 | Train Loss: 7.7272 | Train Loss Unweighted: 2.6824 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3947 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1238 | Regression_eta: 0.0084 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6709 | Classification_binary: 2.3947 | Regression_cos_phi: 0.2990 | Regression_energy: 1.6664 | Regression_eta: 0.3533 | Regression_pt: 1.8049 | Regression_sin_phi: 0.5380 | LR: 1.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4217s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1035 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0342 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.2695 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6145.4 | uss_MB: 5670.6 | child_rss_MB: 3185.6 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:51,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:51,588] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 13:17:51,674] INFO : Running validation on rank0 +rank0 [2026-10-28 13:17:51,674] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:17:51,674] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:17:52,748] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 13:17:52,895] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:17:53,027] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:17:53,103] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:17:53,171] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:17:53,240] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:17:53,311] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:17:53,380] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:17:53,494] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:17:53,619] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:17:53,742] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:17:53,814] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:17:53,884] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:17:53,952] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:17:54,022] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:17:54,093] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:17:54,192] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:17:54,199] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:17:54,201] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:17:54,204] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:17:54,205] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:17:54,207] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:17:54,210] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:17:54,213] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:17:54,214] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:17:54,216] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:17:54,218] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:17:54,219] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:17:54,222] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:17:54,225] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:17:54,227] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:17:54,228] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:17:59,969] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.7272 | Valid Loss=8.1290 | Train Loss Unweighted=2.6824 | Valid Loss Unweighted=2.7163 | Stale=0 | ETA=8.0m +rank0 [2026-10-28 13:17:59,977] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:00,039] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:18:00,043] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:00,228] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:01,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:02,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:02,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:03,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:03,571] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:03,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:03,630] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:18:03,633] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:03,746] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:05,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:05,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:06,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:06,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:07,065] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:07,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:07,121] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:18:07,124] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:18:07,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:08,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:09,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:09,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:10,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:10,600] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:10,608] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:10,857] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:11,314] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:12,765] INFO : Plotted jets +rank0 [2026-10-28 13:18:13,871] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:13,905] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:14,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:14,511] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:15,907] INFO : Plotted jets +rank0 [2026-10-28 13:18:17,545] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:17,578] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 13:18:17,781] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 13:18:18,201] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:19,599] INFO : Plotted jets +rank0 [2026-10-28 13:18:20,698] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:32,134] INFO : Step 47100/50000 rank0 | Train Loss: 7.9423 | Train Loss Unweighted: 2.5714 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2653 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1328 | Regression_eta: 0.0087 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5915 | Classification_binary: 2.2653 | Regression_cos_phi: 0.4266 | Regression_energy: 1.7876 | Regression_eta: 0.3655 | Regression_pt: 1.9319 | Regression_sin_phi: 0.5738 | LR: 1.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.2996 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6343.4 | uss_MB: 5867.4 | child_rss_MB: 3183.9 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:32,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:43,272] INFO : Step 47200/50000 rank0 | Train Loss: 8.1329 | Train Loss Unweighted: 2.7013 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1269 | Regression_eta: 0.0097 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6403 | Classification_binary: 2.4026 | Regression_cos_phi: 0.4725 | Regression_energy: 1.7079 | Regression_eta: 0.4084 | Regression_pt: 1.8475 | Regression_sin_phi: 0.6538 | LR: 1.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4445s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0736 | max_loss: 0.0035 | max_backward: 0.0328 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:43,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:54,390] INFO : Step 47300/50000 rank0 | Train Loss: 8.0177 | Train Loss Unweighted: 2.7304 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4432 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1216 | Regression_eta: 0.0093 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6534 | Classification_binary: 2.4432 | Regression_cos_phi: 0.4185 | Regression_energy: 1.6371 | Regression_eta: 0.3884 | Regression_pt: 1.7595 | Regression_sin_phi: 0.7178 | LR: 1.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4623s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1073 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3172 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3185.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:54,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:05,520] INFO : Step 47400/50000 rank0 | Train Loss: 8.4397 | Train Loss Unweighted: 2.7208 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4022 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1352 | Regression_eta: 0.0096 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6499 | Classification_binary: 2.4022 | Regression_cos_phi: 0.4168 | Regression_energy: 1.8204 | Regression_eta: 0.4003 | Regression_pt: 2.0143 | Regression_sin_phi: 0.7358 | LR: 1.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4501s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0901 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.3035 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:05,732] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:16,665] INFO : Step 47500/50000 rank0 | Train Loss: 8.3159 | Train Loss Unweighted: 2.7724 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4599 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1349 | Regression_eta: 0.0096 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6240 | Classification_binary: 2.4599 | Regression_cos_phi: 0.4146 | Regression_energy: 1.8158 | Regression_eta: 0.4028 | Regression_pt: 1.9595 | Regression_sin_phi: 0.6394 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3710s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0036 | max_backward: 0.0332 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3100 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6218.1 | uss_MB: 5741.1 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:16,879] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:27,715] INFO : Step 47600/50000 rank0 | Train Loss: 7.2314 | Train Loss Unweighted: 2.5732 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1084 | Regression_eta: 0.0075 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6020 | Classification_binary: 2.3173 | Regression_cos_phi: 0.3945 | Regression_energy: 1.4590 | Regression_eta: 0.3123 | Regression_pt: 1.5785 | Regression_sin_phi: 0.5678 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0154 | max_backward: 0.0326 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3681 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6219.1 | uss_MB: 5742.1 | child_rss_MB: 3183.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:27,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:38,930] INFO : Step 47700/50000 rank0 | Train Loss: 8.7205 | Train Loss Unweighted: 2.7929 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4667 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1398 | Regression_eta: 0.0111 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6089 | Classification_binary: 2.4667 | Regression_cos_phi: 0.5170 | Regression_energy: 1.8824 | Regression_eta: 0.4641 | Regression_pt: 2.0236 | Regression_sin_phi: 0.7580 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3059 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 6219.1 | uss_MB: 5742.5 | child_rss_MB: 3182.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:19:39,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:49,975] INFO : Step 47800/50000 rank0 | Train Loss: 8.1948 | Train Loss Unweighted: 2.8052 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5044 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1296 | Regression_eta: 0.0090 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6944 | Classification_binary: 2.5044 | Regression_cos_phi: 0.3961 | Regression_energy: 1.7449 | Regression_eta: 0.3762 | Regression_pt: 1.8501 | Regression_sin_phi: 0.6288 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0093 | max_backward: 0.0338 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.3546 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6219.1 | uss_MB: 5742.5 | child_rss_MB: 3187.1 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:50,190] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:01,122] INFO : Step 47900/50000 rank0 | Train Loss: 8.3513 | Train Loss Unweighted: 2.7511 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4495 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1263 | Regression_eta: 0.0104 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6759 | Classification_binary: 2.4495 | Regression_cos_phi: 0.4568 | Regression_energy: 1.6998 | Regression_eta: 0.4363 | Regression_pt: 1.8551 | Regression_sin_phi: 0.7780 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0300 | max_backward: 0.0338 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3658 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6218.1 | uss_MB: 5742.5 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:20:01,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:12,300] INFO : Step 48000/50000 rank0 | Train Loss: 7.7611 | Train Loss Unweighted: 2.7114 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4356 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1167 | Regression_eta: 0.0083 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6546 | Classification_binary: 2.4356 | Regression_cos_phi: 0.4509 | Regression_energy: 1.5713 | Regression_eta: 0.3478 | Regression_pt: 1.6883 | Regression_sin_phi: 0.6126 | LR: 1.04e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4319s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1066 | max_loss: 0.0035 | max_backward: 0.0392 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6219.1 | uss_MB: 5745.4 | child_rss_MB: 3186.5 | allocated_MB: 210.7 | reserved_MB: 2802.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:20:12,512] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:12,586] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 13:20:12,669] INFO : Running validation on rank0 +rank0 [2026-10-28 13:20:12,669] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:20:12,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:20:13,725] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 13:20:13,870] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:20:14,002] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:20:14,076] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:20:14,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:20:14,213] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:20:14,282] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:20:14,351] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:20:14,464] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:20:14,597] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:20:14,729] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:20:14,801] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:20:14,870] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:20:14,938] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:20:15,007] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:20:15,077] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:20:15,176] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:20:15,182] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:20:15,184] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:20:15,185] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:20:15,188] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:20:15,189] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:20:15,192] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:20:15,194] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:20:15,195] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:20:15,198] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:20:15,201] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:20:15,203] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:20:15,204] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:20:15,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:20:15,224] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:20:15,227] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:20:20,936] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.7611 | Valid Loss=8.1381 | Train Loss Unweighted=2.7114 | Valid Loss Unweighted=2.7130 | Stale=1 | ETA=5.3m +rank0 [2026-10-28 13:20:20,944] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:21,006] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:20:21,010] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:21,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:22,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:23,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:23,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:24,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:24,520] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:20:24,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:24,579] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:20:24,582] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:24,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:26,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:26,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:27,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:27,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:28,100] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:20:28,103] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:28,156] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:20:28,159] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:20:28,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:29,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:30,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:30,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:31,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:31,616] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:20:31,623] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:31,875] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:32,330] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:33,770] INFO : Plotted jets +rank0 [2026-10-28 13:20:34,883] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:34,915] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:35,143] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:35,522] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:36,910] INFO : Plotted jets +rank0 [2026-10-28 13:20:38,598] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:38,630] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 13:20:38,831] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 13:20:39,249] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:40,643] INFO : Plotted jets +rank0 [2026-10-28 13:20:41,749] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:53,111] INFO : Step 48100/50000 rank0 | Train Loss: 7.7659 | Train Loss Unweighted: 2.6934 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.4196 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1154 | Regression_eta: 0.0094 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5734 | Classification_binary: 2.4196 | Regression_cos_phi: 0.4520 | Regression_energy: 1.5541 | Regression_eta: 0.3943 | Regression_pt: 1.6828 | Regression_sin_phi: 0.6898 | LR: 1.03e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4563s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0046 | max_backward: 0.0343 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6406.7 | uss_MB: 5931.6 | child_rss_MB: 3184.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:20:53,324] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:04,656] INFO : Step 48200/50000 rank0 | Train Loss: 7.8099 | Train Loss Unweighted: 2.6632 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1259 | Regression_eta: 0.0077 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6089 | Classification_binary: 2.3723 | Regression_cos_phi: 0.3881 | Regression_energy: 1.6955 | Regression_eta: 0.3240 | Regression_pt: 1.8223 | Regression_sin_phi: 0.5987 | LR: 1.03e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.1386 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6406.7 | uss_MB: 5931.6 | child_rss_MB: 3211.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:04,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:15,994] INFO : Step 48300/50000 rank0 | Train Loss: 7.5406 | Train Loss Unweighted: 2.6284 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3625 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1113 | Regression_eta: 0.0096 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5984 | Classification_binary: 2.3625 | Regression_cos_phi: 0.3951 | Regression_energy: 1.4990 | Regression_eta: 0.4022 | Regression_pt: 1.6371 | Regression_sin_phi: 0.6463 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.1058 | max_backward: 0.0340 | max_optimizer_step: 0.1429 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6406.7 | uss_MB: 5931.7 | child_rss_MB: 3184.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:16,204] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:27,175] INFO : Step 48400/50000 rank0 | Train Loss: 7.8456 | Train Loss Unweighted: 2.6842 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4080 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1136 | Regression_eta: 0.0102 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.4080 | Regression_cos_phi: 0.5257 | Regression_energy: 1.5294 | Regression_eta: 0.4255 | Regression_pt: 1.6725 | Regression_sin_phi: 0.5623 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1095 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6406.7 | uss_MB: 5931.7 | child_rss_MB: 3187.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:21:27,394] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:38,353] INFO : Step 48500/50000 rank0 | Train Loss: 7.7940 | Train Loss Unweighted: 2.6853 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4136 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1140 | Regression_eta: 0.0082 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5911 | Classification_binary: 2.4136 | Regression_cos_phi: 0.5096 | Regression_energy: 1.5349 | Regression_eta: 0.3455 | Regression_pt: 1.6572 | Regression_sin_phi: 0.7420 | LR: 1.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4325s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0892 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1399 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6291.3 | uss_MB: 5815.9 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:38,570] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:49,470] INFO : Step 48600/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.6888 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3816 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1264 | Regression_eta: 0.0200 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6609 | Classification_binary: 2.3816 | Regression_cos_phi: 0.4993 | Regression_energy: 1.7016 | Regression_eta: 0.8389 | Regression_pt: 1.8090 | Regression_sin_phi: 0.6690 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3793s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0382 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1411 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6291.3 | uss_MB: 5817.0 | child_rss_MB: 3187.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:49,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:00,590] INFO : Step 48700/50000 rank0 | Train Loss: 8.3131 | Train Loss Unweighted: 2.6722 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3677 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1266 | Regression_eta: 0.0111 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6811 | Classification_binary: 2.3677 | Regression_cos_phi: 0.5424 | Regression_energy: 1.7049 | Regression_eta: 0.4662 | Regression_pt: 1.8740 | Regression_sin_phi: 0.6768 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0869 | max_forward: 0.0181 | max_loss: 0.0037 | max_backward: 0.0390 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6291.3 | uss_MB: 5819.2 | child_rss_MB: 3187.0 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:22:00,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:11,585] INFO : Step 48800/50000 rank0 | Train Loss: 7.8636 | Train Loss Unweighted: 2.5751 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.2762 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1298 | Regression_eta: 0.0068 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5841 | Classification_binary: 2.2762 | Regression_cos_phi: 0.4613 | Regression_energy: 1.7470 | Regression_eta: 0.2852 | Regression_pt: 1.8829 | Regression_sin_phi: 0.6270 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0577 | max_loss: 0.0036 | max_backward: 0.0328 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3223 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6291.3 | uss_MB: 5817.5 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:11,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:22,699] INFO : Step 48900/50000 rank0 | Train Loss: 7.7267 | Train Loss Unweighted: 2.5843 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3012 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1202 | Regression_eta: 0.0093 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6047 | Classification_binary: 2.3012 | Regression_cos_phi: 0.4087 | Regression_energy: 1.6188 | Regression_eta: 0.3908 | Regression_pt: 1.7556 | Regression_sin_phi: 0.6468 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4416s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0862 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3240 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6291.3 | uss_MB: 5817.6 | child_rss_MB: 3187.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:22,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:33,731] INFO : Step 49000/50000 rank0 | Train Loss: 7.5248 | Train Loss Unweighted: 2.6023 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3189 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1233 | Regression_eta: 0.0080 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5841 | Classification_binary: 2.3189 | Regression_cos_phi: 0.3602 | Regression_energy: 1.6599 | Regression_eta: 0.3363 | Regression_pt: 1.7885 | Regression_sin_phi: 0.4769 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0060 | max_backward: 0.0337 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3706 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6291.3 | uss_MB: 5817.7 | child_rss_MB: 3186.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:22:33,947] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:33,963] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 13:22:34,164] INFO : Running validation on rank0 +rank0 [2026-10-28 13:22:34,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:22:34,165] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:22:35,269] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 13:22:35,414] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:22:35,545] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:22:35,619] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:22:35,688] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:22:35,755] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:22:35,825] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:22:35,895] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:22:36,007] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:22:36,141] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:22:36,273] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:22:36,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:22:36,414] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:22:36,482] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:22:36,552] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:22:36,622] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:22:36,720] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:22:36,726] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:22:36,729] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:22:36,730] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:22:36,733] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:22:36,735] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:22:36,736] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:22:36,739] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:22:36,741] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:22:36,743] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:22:36,745] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:22:36,748] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:22:36,749] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:22:36,751] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:22:36,754] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:22:36,755] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:22:42,519] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.5248 | Valid Loss=8.0991 | Train Loss Unweighted=2.6023 | Valid Loss Unweighted=2.7077 | Stale=0 | ETA=2.7m +rank0 [2026-10-28 13:22:42,529] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:42,590] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:22:42,594] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:42,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:44,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:45,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:45,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:46,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:46,613] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:46,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:46,674] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:22:46,677] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:46,817] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:48,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:48,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:49,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:50,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:50,275] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:50,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:50,332] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:22:50,335] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:22:50,468] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:52,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:52,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:53,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:53,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:53,953] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:53,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:54,212] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:54,673] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:56,120] INFO : Plotted jets +rank0 [2026-10-28 13:22:57,230] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:57,263] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:57,491] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:57,872] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:59,266] INFO : Plotted jets +rank0 [2026-10-28 13:23:00,888] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:00,921] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 13:23:01,124] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 13:23:01,544] INFO : Plotted number of elements +rank0 [2026-10-28 13:23:02,945] INFO : Plotted jets +rank0 [2026-10-28 13:23:04,052] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:15,375] INFO : Step 49100/50000 rank0 | Train Loss: 8.4073 | Train Loss Unweighted: 2.7947 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4959 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1254 | Regression_eta: 0.0123 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6280 | Classification_binary: 2.4959 | Regression_cos_phi: 0.4788 | Regression_energy: 1.6888 | Regression_eta: 0.5168 | Regression_pt: 1.8102 | Regression_sin_phi: 0.7889 | LR: 1.01e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0418 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2716 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3190.8 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:15,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:26,472] INFO : Step 49200/50000 rank0 | Train Loss: 8.4810 | Train Loss Unweighted: 2.7082 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4018 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1278 | Regression_eta: 0.0125 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6379 | Classification_binary: 2.4018 | Regression_cos_phi: 0.5721 | Regression_energy: 1.7203 | Regression_eta: 0.5249 | Regression_pt: 1.8606 | Regression_sin_phi: 0.7633 | LR: 1.01e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0982 | max_forward: 0.0160 | max_loss: 0.0041 | max_backward: 0.0339 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2738 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:23:26,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:37,504] INFO : Step 49300/50000 rank0 | Train Loss: 7.6860 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4217 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1142 | Regression_eta: 0.0085 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6251 | Classification_binary: 2.4217 | Regression_cos_phi: 0.3375 | Regression_energy: 1.5371 | Regression_eta: 0.3577 | Regression_pt: 1.6928 | Regression_sin_phi: 0.7141 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.2959 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:23:37,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:48,602] INFO : Step 49400/50000 rank0 | Train Loss: 7.6091 | Train Loss Unweighted: 2.5949 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3211 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1145 | Regression_eta: 0.0086 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6204 | Classification_binary: 2.3211 | Regression_cos_phi: 0.4012 | Regression_energy: 1.5413 | Regression_eta: 0.3598 | Regression_pt: 1.7089 | Regression_sin_phi: 0.6564 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1013 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6478.5 | uss_MB: 6007.2 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:48,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:59,736] INFO : Step 49500/50000 rank0 | Train Loss: 7.5559 | Train Loss Unweighted: 2.5350 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2601 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1169 | Regression_eta: 0.0099 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5858 | Classification_binary: 2.2601 | Regression_cos_phi: 0.4476 | Regression_energy: 1.5743 | Regression_eta: 0.4159 | Regression_pt: 1.6843 | Regression_sin_phi: 0.5880 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0345 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6478.5 | uss_MB: 6007.2 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:59,949] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:10,807] INFO : Step 49600/50000 rank0 | Train Loss: 8.0050 | Train Loss Unweighted: 2.6557 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3509 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1338 | Regression_eta: 0.0094 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6090 | Classification_binary: 2.3509 | Regression_cos_phi: 0.3674 | Regression_energy: 1.8017 | Regression_eta: 0.3950 | Regression_pt: 1.8921 | Regression_sin_phi: 0.5888 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4298s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0631 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6361.6 | uss_MB: 5892.8 | child_rss_MB: 3194.0 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:24:11,021] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:22,005] INFO : Step 49700/50000 rank0 | Train Loss: 7.9377 | Train Loss Unweighted: 2.6360 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3415 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1260 | Regression_eta: 0.0085 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6018 | Classification_binary: 2.3415 | Regression_cos_phi: 0.5167 | Regression_energy: 1.6969 | Regression_eta: 0.3571 | Regression_pt: 1.8301 | Regression_sin_phi: 0.5936 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0639 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1438 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 6361.6 | uss_MB: 5891.2 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:24:22,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:33,261] INFO : Step 49800/50000 rank0 | Train Loss: 8.4460 | Train Loss Unweighted: 2.7490 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.4432 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1272 | Regression_eta: 0.0128 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7289 | Classification_binary: 2.4432 | Regression_cos_phi: 0.4549 | Regression_energy: 1.7122 | Regression_eta: 0.5367 | Regression_pt: 1.8568 | Regression_sin_phi: 0.7133 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3658s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6361.6 | uss_MB: 5891.4 | child_rss_MB: 3190.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:33,474] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:44,482] INFO : Step 49900/50000 rank0 | Train Loss: 8.0574 | Train Loss Unweighted: 2.7208 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4266 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1258 | Regression_eta: 0.0098 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5980 | Classification_binary: 2.4266 | Regression_cos_phi: 0.4255 | Regression_energy: 1.6932 | Regression_eta: 0.4093 | Regression_pt: 1.8224 | Regression_sin_phi: 0.6823 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0160 | max_loss: 0.0035 | max_backward: 0.1391 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6361.6 | uss_MB: 5891.5 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:44,696] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:55,759] INFO : Step 50000/50000 rank0 | Train Loss: 7.7472 | Train Loss Unweighted: 2.6151 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3250 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1245 | Regression_eta: 0.0077 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5380 | Classification_binary: 2.3250 | Regression_cos_phi: 0.4189 | Regression_energy: 1.6767 | Regression_eta: 0.3211 | Regression_pt: 1.8501 | Regression_sin_phi: 0.6174 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0638 | max_optimizer_step: 0.1436 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6361.6 | uss_MB: 5890.4 | child_rss_MB: 3190.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:55,971] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:55,988] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 13:24:56,070] INFO : Running validation on rank0 +rank0 [2026-10-28 13:24:56,070] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:24:56,070] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:24:57,147] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 13:24:57,293] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:24:57,424] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:24:57,499] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:24:57,566] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:24:57,635] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:24:57,705] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:24:57,774] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:24:57,887] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:24:58,020] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:24:58,152] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:24:58,223] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:24:58,293] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:24:58,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:24:58,433] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:24:58,503] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:24:58,602] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:24:58,608] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:24:58,611] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:24:58,614] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:24:58,615] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:24:58,617] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:24:58,620] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:24:58,621] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:24:58,624] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:24:58,626] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:24:58,627] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:24:58,630] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:24:58,633] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:24:58,633] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:24:58,636] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:24:58,639] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:25:04,409] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.7472 | Valid Loss=8.0861 | Train Loss Unweighted=2.6151 | Valid Loss Unweighted=2.7040 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 13:25:04,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:04,481] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:25:04,485] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:04,598] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:06,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:06,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:07,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:07,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:08,002] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:25:08,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:08,061] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:25:08,079] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:08,280] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:09,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:10,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:10,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:11,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:11,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:25:11,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:11,679] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:25:11,681] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:25:11,817] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:13,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:13,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:14,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:14,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:15,244] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:25:15,252] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:15,508] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:15,968] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:17,411] INFO : Plotted jets +rank0 [2026-10-28 13:25:18,537] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:18,570] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:18,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:19,175] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:20,568] INFO : Plotted jets +rank0 [2026-10-28 13:25:22,221] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:22,254] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 13:25:22,456] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 13:25:22,873] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:24,275] INFO : Plotted jets +rank0 [2026-10-28 13:25:25,376] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:25,866] INFO : Training completed. Total time on device 0: 132.873min diff --git a/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log.0 b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..a2570784f893a72324f4c5a8d2e10f73e4f46e08 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_attention_seed12345_20261028_111202_052283/train.log.0 @@ -0,0 +1,13847 @@ +rank0 [2026-10-28 11:12:24,881] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,882] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,882] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,882] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,883] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,919] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,919] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,919] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,919] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,922] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,922] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,925] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,925] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,928] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,928] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,931] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:24,931] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,934] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:24,934] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-10-28 11:12:24,936] INFO : Backbone initialization took 0.02s +rank0 [2026-10-28 11:12:24,937] INFO : backbone parameters: 2374656 +rank0 [2026-10-28 11:12:24,937] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:24,941] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:24,942] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:24,942] INFO : jagged_attention enabled=True +rank0 [2026-10-28 11:12:24,942] INFO : flash_attn_varlen enabled=True +rank0 [2026-10-28 11:12:24,942] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:24,942] INFO : classification_norm parameters: 512 +rank0 [2026-10-28 11:12:24,942] INFO : regression_norm parameters: 512 +rank0 [2026-10-28 11:12:24,942] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:24,942] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:24,942] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:24,943] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:24,943] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:24,943] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:24,943] INFO : Total MLPF parameters: 3114258 +rank0 [2026-10-28 11:12:24,944] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:24,944] INFO : Instantiated model +rank0 [2026-10-28 11:12:24,945] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,169] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,170] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,249] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +131 nn_energy.nn.1.0.bias 256 0 +132 nn_energy.nn.1.2.weight 256 0 +133 nn_energy.nn.1.2.bias 256 0 +134 nn_energy.nn.1.4.weight 256 0 +135 nn_energy.nn.1.4.bias 1 0 + +[136 rows x 3 columns] +rank0 [2026-10-28 11:12:25,249] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,251] INFO : Trainable parameters: 3114258 +rank0 [2026-10-28 11:12:25,251] INFO : Non-trainable parameters: 0 +rank0 [2026-10-28 11:12:25,251] INFO : Total parameters: 3114258 +rank0 [2026-10-28 11:12:25,252] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 +backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 +backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 +backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 +backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 +backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 +backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,256] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283 +rank0 [2026-10-28 11:12:25,256] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283 +rank0 [2026-10-28 11:12:25,256] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,256] INFO : sample=clic_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,313] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,355] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,395] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,416] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,473] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,512] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,549] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,591] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,614] INFO : train_dataset: clic_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,614] INFO : sample=clic_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,650] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,679] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,704] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,744] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,783] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,814] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,842] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,876] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,900] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,935] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,935] INFO : sample=clic_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,973] INFO : train_dataset: clic_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,012] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,082] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,102] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,123] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,172] INFO : train_dataset: clic_edm_qq_pf, 89996 +rank0 [2026-10-28 11:12:33,199] INFO : train_dataset: clic_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:33,229] INFO : train_dataset: clic_edm_qq_pf, 89994 +rank0 [2026-10-28 11:12:33,262] INFO : train_dataset: clic_edm_qq_pf, 89995 +rank0 [2026-10-28 11:12:33,277] INFO : train_dataset: clic_edm_qq_pf, 90099 +rank0 [2026-10-28 11:12:33,277] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,277] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,277] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,277] INFO : sample=clic_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,283] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,289] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,294] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,300] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,306] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,312] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,318] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,324] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,330] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,336] INFO : sample=clic_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,341] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,346] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,352] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,357] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,362] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,367] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,373] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,378] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,383] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : sample=clic_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,393] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,398] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,403] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,409] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,414] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,419] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,424] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,429] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,434] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,439] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,475] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,478] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,478] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:12:33,478] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,478] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,484] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,502] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,757] INFO : Created train_iterator +rank0 [2026-10-28 11:13:38,245] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 4.923548955917358, 'Classification': 0.14500824306160212, 'Regression_pt': 0.33665568113327027, 'Regression_eta': 0.1174755490757525, 'Regression_sin_phi': 0.06220133699476719, 'Regression_cos_phi': 0.09051778480410576, 'Regression_energy': 0.3657260942459106}, weights={'Classification_binary': 1.0, 'Classification': 33.95357894897461, 'Regression_pt': 14.624879837036133, 'Regression_eta': 41.9112663269043, 'Regression_sin_phi': 79.15503692626953, 'Regression_cos_phi': 54.393165588378906, 'Regression_energy': 13.462394714355469} +rank0 [2026-10-28 11:13:40,247] INFO : Step 100/50000 rank0 | Train Loss: 5.4392 | Train Loss Unweighted: 5.4392 | Train Loss Components: Classification: 0.0536 | Classification_binary: 4.6139 | Regression_cos_phi: 0.0261 | Regression_energy: 0.3386 | Regression_eta: 0.0561 | Regression_pt: 0.3237 | Regression_sin_phi: 0.0273 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0536 | Classification_binary: 4.6139 | Regression_cos_phi: 0.0261 | Regression_energy: 0.3386 | Regression_eta: 0.0561 | Regression_pt: 0.3237 | Regression_sin_phi: 0.0273 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 2.3315s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.1049 | max_backward: 0.0335 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 1.0047 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2690.5 | uss_MB: 1982.7 | child_rss_MB: 3287.7 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:13:40,463] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 110.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:02,617] INFO : Step 200/50000 rank0 | Train Loss: 18.1629 | Train Loss Unweighted: 5.1827 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.4838 | Regression_cos_phi: 0.0210 | Regression_energy: 0.2930 | Regression_eta: 0.0264 | Regression_pt: 0.2925 | Regression_sin_phi: 0.0214 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.5144 | Classification_binary: 4.4838 | Regression_cos_phi: 1.1427 | Regression_energy: 3.9443 | Regression_eta: 1.1047 | Regression_pt: 4.2777 | Regression_sin_phi: 1.6953 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 1.1213s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0147 | max_loss: 0.0030 | max_backward: 0.1406 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2653 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2689.8 | uss_MB: 1984.5 | child_rss_MB: 3290.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:14:02,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:23,600] INFO : Step 300/50000 rank0 | Train Loss: 17.0852 | Train Loss Unweighted: 5.0103 | Train Loss Components: Classification: 0.0427 | Classification_binary: 4.3273 | Regression_cos_phi: 0.0175 | Regression_energy: 0.2916 | Regression_eta: 0.0183 | Regression_pt: 0.2961 | Regression_sin_phi: 0.0169 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4486 | Classification_binary: 4.3273 | Regression_cos_phi: 0.9508 | Regression_energy: 3.9258 | Regression_eta: 0.7682 | Regression_pt: 4.3300 | Regression_sin_phi: 1.3346 | LR: 1.00e-03 | DataLoad Time: 0.0013s | Model Forward Time: 1.0807s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0029 | max_backward: 0.0340 | max_optimizer_step: 0.1593 | max_tensorboard_logging: 0.2690 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2689.7 | uss_MB: 1985.1 | child_rss_MB: 3285.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:14:23,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:43,172] INFO : Step 400/50000 rank0 | Train Loss: 15.0521 | Train Loss Unweighted: 4.8845 | Train Loss Components: Classification: 0.0439 | Classification_binary: 4.3147 | Regression_cos_phi: 0.0139 | Regression_energy: 0.2395 | Regression_eta: 0.0172 | Regression_pt: 0.2428 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4900 | Classification_binary: 4.3147 | Regression_cos_phi: 0.7560 | Regression_energy: 3.2239 | Regression_eta: 0.7198 | Regression_pt: 3.5505 | Regression_sin_phi: 0.9971 | LR: 1.00e-03 | DataLoad Time: 0.1810s | Model Forward Time: 1.0707s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0156 | max_loss: 0.0029 | max_backward: 0.0821 | max_optimizer_step: 0.1022 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2691.5 | uss_MB: 1987.6 | child_rss_MB: 3283.6 | allocated_MB: 210.7 | reserved_MB: 2744.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:14:43,387] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:02,383] INFO : Step 500/50000 rank0 | Train Loss: 15.7330 | Train Loss Unweighted: 4.8445 | Train Loss Components: Classification: 0.0406 | Classification_binary: 4.2336 | Regression_cos_phi: 0.0142 | Regression_energy: 0.2630 | Regression_eta: 0.0173 | Regression_pt: 0.2594 | Regression_sin_phi: 0.0163 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3787 | Classification_binary: 4.2336 | Regression_cos_phi: 0.7742 | Regression_energy: 3.5408 | Regression_eta: 0.7241 | Regression_pt: 3.7943 | Regression_sin_phi: 1.2874 | LR: 1.00e-03 | DataLoad Time: 0.0584s | Model Forward Time: 1.1432s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0495 | max_forward: 0.0136 | max_loss: 0.0028 | max_backward: 0.0327 | max_optimizer_step: 0.0972 | max_tensorboard_logging: 0.3586 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2689.1 | uss_MB: 1985.7 | child_rss_MB: 3279.8 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:15:02,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:21,393] INFO : Step 600/50000 rank0 | Train Loss: 16.9350 | Train Loss Unweighted: 5.0748 | Train Loss Components: Classification: 0.0427 | Classification_binary: 4.4156 | Regression_cos_phi: 0.0167 | Regression_energy: 0.2808 | Regression_eta: 0.0234 | Regression_pt: 0.2789 | Regression_sin_phi: 0.0167 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4500 | Classification_binary: 4.4156 | Regression_cos_phi: 0.9108 | Regression_energy: 3.7802 | Regression_eta: 0.9792 | Regression_pt: 4.0790 | Regression_sin_phi: 1.3202 | LR: 1.00e-03 | DataLoad Time: 0.0014s | Model Forward Time: 1.3646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1218 | max_loss: 0.0029 | max_backward: 0.0336 | max_optimizer_step: 0.0970 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 2626.5 | uss_MB: 1922.6 | child_rss_MB: 3276.4 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:15:21,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:40,504] INFO : Step 700/50000 rank0 | Train Loss: 15.3225 | Train Loss Unweighted: 4.8653 | Train Loss Components: Classification: 0.0413 | Classification_binary: 4.2867 | Regression_cos_phi: 0.0158 | Regression_energy: 0.2475 | Regression_eta: 0.0162 | Regression_pt: 0.2425 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4008 | Classification_binary: 4.2867 | Regression_cos_phi: 0.8587 | Regression_energy: 3.3323 | Regression_eta: 0.6772 | Regression_pt: 3.5462 | Regression_sin_phi: 1.2206 | LR: 1.00e-03 | DataLoad Time: 0.0013s | Model Forward Time: 1.0756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.0337 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.2630 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2629.6 | uss_MB: 1926.4 | child_rss_MB: 3278.1 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:15:40,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:59,496] INFO : Step 800/50000 rank0 | Train Loss: 15.0226 | Train Loss Unweighted: 4.8587 | Train Loss Components: Classification: 0.0399 | Classification_binary: 4.2805 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2470 | Regression_eta: 0.0149 | Regression_pt: 0.2497 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3533 | Classification_binary: 4.2805 | Regression_cos_phi: 0.7294 | Regression_energy: 3.3248 | Regression_eta: 0.6265 | Regression_pt: 3.6512 | Regression_sin_phi: 1.0569 | LR: 9.99e-04 | DataLoad Time: 0.0179s | Model Forward Time: 1.1970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0155 | max_loss: 0.0040 | max_backward: 0.0332 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.2703 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2629.6 | uss_MB: 1927.4 | child_rss_MB: 3276.9 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:15:59,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:18,014] INFO : Step 900/50000 rank0 | Train Loss: 14.4804 | Train Loss Unweighted: 4.7851 | Train Loss Components: Classification: 0.0420 | Classification_binary: 4.2489 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2265 | Regression_eta: 0.0172 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4273 | Classification_binary: 4.2489 | Regression_cos_phi: 0.6153 | Regression_energy: 3.0486 | Regression_eta: 0.7199 | Regression_pt: 3.2900 | Regression_sin_phi: 1.1304 | LR: 9.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 0.9911s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0132 | max_loss: 0.0031 | max_backward: 0.0326 | max_optimizer_step: 0.0959 | max_tensorboard_logging: 0.2582 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 2630.0 | uss_MB: 1929.1 | child_rss_MB: 3272.5 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:16:18,227] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:36,500] INFO : Step 1000/50000 rank0 | Train Loss: 14.1550 | Train Loss Unweighted: 4.7953 | Train Loss Components: Classification: 0.0389 | Classification_binary: 4.2751 | Regression_cos_phi: 0.0130 | Regression_energy: 0.2193 | Regression_eta: 0.0165 | Regression_pt: 0.2198 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 4.2751 | Regression_cos_phi: 0.7094 | Regression_energy: 2.9529 | Regression_eta: 0.6901 | Regression_pt: 3.2151 | Regression_sin_phi: 0.9906 | LR: 9.99e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1159s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.1418 | max_optimizer_step: 0.0990 | max_tensorboard_logging: 0.2606 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2630.5 | uss_MB: 1931.3 | child_rss_MB: 3267.2 | allocated_MB: 207.0 | reserved_MB: 2744.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:16:36,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:36,743] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:16:36,824] INFO : Running validation on rank0 +rank0 [2026-10-28 11:16:36,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:16:36,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:16:38,130] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:16:38,316] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:16:38,413] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:16:38,473] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:16:38,534] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:16:38,584] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:16:38,645] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:16:38,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:16:38,761] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:16:38,847] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:16:38,934] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:16:38,993] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:16:39,053] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:16:39,102] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:16:39,163] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:16:39,225] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:16:39,283] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:16:39,287] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:16:39,290] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:16:39,293] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:16:39,295] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:16:39,298] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:16:39,299] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:16:39,302] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:16:39,304] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:16:39,305] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:16:39,308] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:16:39,310] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:16:39,313] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:16:39,314] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:16:39,316] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:16:39,319] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:16:44,918] INFO : VALIDATION | Step=1000/50000 | Train Loss=14.1550 | Valid Loss=14.8413 | Train Loss Unweighted=4.7953 | Valid Loss Unweighted=4.7426 | Stale=0 | ETA=205.3m +rank0 [2026-10-28 11:16:44,926] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:44,988] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:16:44,998] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:45,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:47,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:48,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:48,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:49,531] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:49,593] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:16:49,595] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:49,652] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:16:49,655] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:16:49,706] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:51,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:51,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:52,302] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:52,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:52,904] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:52,907] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:16:52,962] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:16:52,965] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:16:53,035] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:16:54,384] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:16:54,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:16:55,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:16:55,918] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:16:55,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:16:56,005] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:56,261] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:16:57,235] INFO : Plotted number of elements +rank0 [2026-10-28 11:16:58,689] INFO : Plotted jets +rank0 [2026-10-28 11:16:59,769] INFO : Plotted jet ratio +rank0 [2026-10-28 11:16:59,801] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:00,028] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:17:00,401] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:01,776] INFO : Plotted jets +rank0 [2026-10-28 11:17:02,880] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:02,913] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:17:03,112] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:17:03,830] INFO : Plotted number of elements +rank0 [2026-10-28 11:17:05,203] INFO : Plotted jets +rank0 [2026-10-28 11:17:06,278] INFO : Plotted jet ratio +rank0 [2026-10-28 11:17:24,611] INFO : Step 1100/50000 rank0 | Train Loss: 15.1176 | Train Loss Unweighted: 4.6801 | Train Loss Components: Classification: 0.0426 | Classification_binary: 4.0828 | Regression_cos_phi: 0.0140 | Regression_energy: 0.2556 | Regression_eta: 0.0154 | Regression_pt: 0.2574 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4479 | Classification_binary: 4.0828 | Regression_cos_phi: 0.7597 | Regression_energy: 3.4407 | Regression_eta: 0.6470 | Regression_pt: 3.7646 | Regression_sin_phi: 0.9749 | LR: 9.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2608 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2905.5 | uss_MB: 2392.2 | child_rss_MB: 3266.3 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:17:24,823] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:43,392] INFO : Step 1200/50000 rank0 | Train Loss: 15.1457 | Train Loss Unweighted: 4.6590 | Train Loss Components: Classification: 0.0389 | Classification_binary: 4.0706 | Regression_cos_phi: 0.0156 | Regression_energy: 0.2529 | Regression_eta: 0.0136 | Regression_pt: 0.2516 | Regression_sin_phi: 0.0158 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 4.0706 | Regression_cos_phi: 0.8468 | Regression_energy: 3.4048 | Regression_eta: 0.5712 | Regression_pt: 3.6790 | Regression_sin_phi: 1.2517 | LR: 9.99e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0417 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3703 | max_tensorboard_flush: 0.0147 | Memory: rss_MB: 2906.5 | uss_MB: 2393.1 | child_rss_MB: 3262.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:17:43,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:02,122] INFO : Step 1300/50000 rank0 | Train Loss: 15.2010 | Train Loss Unweighted: 4.7283 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.1439 | Regression_cos_phi: 0.0153 | Regression_energy: 0.2524 | Regression_eta: 0.0143 | Regression_pt: 0.2453 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4147 | Classification_binary: 4.1439 | Regression_cos_phi: 0.8310 | Regression_energy: 3.3984 | Regression_eta: 0.6009 | Regression_pt: 3.5873 | Regression_sin_phi: 1.2249 | LR: 9.98e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0519 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1036 | max_tensorboard_logging: 0.3592 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2907.3 | uss_MB: 2393.7 | child_rss_MB: 3262.1 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:02,334] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:20,704] INFO : Step 1400/50000 rank0 | Train Loss: 14.2392 | Train Loss Unweighted: 4.6683 | Train Loss Components: Classification: 0.0386 | Classification_binary: 4.1235 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2344 | Regression_eta: 0.0124 | Regression_pt: 0.2339 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3110 | Classification_binary: 4.1235 | Regression_cos_phi: 0.6622 | Regression_energy: 3.1555 | Regression_eta: 0.5217 | Regression_pt: 3.4212 | Regression_sin_phi: 1.0440 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1455s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.1389 | max_optimizer_step: 0.1020 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2907.3 | uss_MB: 2394.2 | child_rss_MB: 3263.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:20,931] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:38,932] INFO : Step 1500/50000 rank0 | Train Loss: 14.0094 | Train Loss Unweighted: 4.4789 | Train Loss Components: Classification: 0.0390 | Classification_binary: 3.9469 | Regression_cos_phi: 0.0135 | Regression_energy: 0.2247 | Regression_eta: 0.0146 | Regression_pt: 0.2271 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3239 | Classification_binary: 3.9469 | Regression_cos_phi: 0.7322 | Regression_energy: 3.0243 | Regression_eta: 0.6110 | Regression_pt: 3.3209 | Regression_sin_phi: 1.0501 | LR: 9.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1052s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0163 | max_loss: 0.0727 | max_backward: 0.0339 | max_optimizer_step: 0.0992 | max_tensorboard_logging: 0.2714 | max_tensorboard_flush: 0.0131 | Memory: rss_MB: 2908.3 | uss_MB: 2394.8 | child_rss_MB: 3289.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:18:39,144] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:57,280] INFO : Step 1600/50000 rank0 | Train Loss: 14.5911 | Train Loss Unweighted: 4.5934 | Train Loss Components: Classification: 0.0378 | Classification_binary: 4.0348 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2408 | Regression_eta: 0.0123 | Regression_pt: 0.2365 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2836 | Classification_binary: 4.0348 | Regression_cos_phi: 0.8999 | Regression_energy: 3.2415 | Regression_eta: 0.5171 | Regression_pt: 3.4581 | Regression_sin_phi: 1.1561 | LR: 9.98e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1902s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0807 | max_loss: 0.0032 | max_backward: 0.0324 | max_optimizer_step: 0.0997 | max_tensorboard_logging: 0.3736 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 2792.1 | uss_MB: 2280.0 | child_rss_MB: 3260.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:18:57,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:15,526] INFO : Step 1700/50000 rank0 | Train Loss: 13.7400 | Train Loss Unweighted: 4.4868 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.9588 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2282 | Regression_eta: 0.0106 | Regression_pt: 0.2255 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3217 | Classification_binary: 3.9588 | Regression_cos_phi: 0.6881 | Regression_energy: 3.0726 | Regression_eta: 0.4453 | Regression_pt: 3.2978 | Regression_sin_phi: 0.9556 | LR: 9.97e-04 | DataLoad Time: 0.0337s | Model Forward Time: 1.2122s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0930 | max_forward: 0.0147 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1012 | max_tensorboard_logging: 0.3533 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2793.1 | uss_MB: 2283.0 | child_rss_MB: 3259.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:19:15,738] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:33,648] INFO : Step 1800/50000 rank0 | Train Loss: 14.2339 | Train Loss Unweighted: 4.4368 | Train Loss Components: Classification: 0.0392 | Classification_binary: 3.8847 | Regression_cos_phi: 0.0131 | Regression_energy: 0.2364 | Regression_eta: 0.0158 | Regression_pt: 0.2346 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3311 | Classification_binary: 3.8847 | Regression_cos_phi: 0.7107 | Regression_energy: 3.1821 | Regression_eta: 0.6614 | Regression_pt: 3.4305 | Regression_sin_phi: 1.0332 | LR: 9.97e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2048s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1081 | max_loss: 0.0029 | max_backward: 0.0335 | max_optimizer_step: 0.1010 | max_tensorboard_logging: 0.3605 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 2793.1 | uss_MB: 2283.2 | child_rss_MB: 3259.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:33,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:51,864] INFO : Step 1900/50000 rank0 | Train Loss: 14.1379 | Train Loss Unweighted: 4.5553 | Train Loss Components: Classification: 0.0416 | Classification_binary: 4.0280 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2180 | Regression_eta: 0.0125 | Regression_pt: 0.2261 | Regression_sin_phi: 0.0141 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.4130 | Classification_binary: 4.0280 | Regression_cos_phi: 0.8175 | Regression_energy: 2.9346 | Regression_eta: 0.5243 | Regression_pt: 3.3063 | Regression_sin_phi: 1.1142 | LR: 9.97e-04 | DataLoad Time: 0.1309s | Model Forward Time: 1.2568s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1231 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1072 | max_tensorboard_logging: 0.3578 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2794.1 | uss_MB: 2283.7 | child_rss_MB: 3256.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:19:52,079] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 118.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:10,326] INFO : Step 2000/50000 rank0 | Train Loss: 13.6430 | Train Loss Unweighted: 4.4829 | Train Loss Components: Classification: 0.0374 | Classification_binary: 3.9654 | Regression_cos_phi: 0.0119 | Regression_energy: 0.2202 | Regression_eta: 0.0164 | Regression_pt: 0.2205 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2688 | Classification_binary: 3.9654 | Regression_cos_phi: 0.6460 | Regression_energy: 2.9643 | Regression_eta: 0.6872 | Regression_pt: 3.2254 | Regression_sin_phi: 0.8858 | LR: 9.96e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2159s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0833 | max_forward: 0.0164 | max_loss: 0.0029 | max_backward: 0.0342 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.3554 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 2794.1 | uss_MB: 2284.5 | child_rss_MB: 3257.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:20:10,541] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:20:10,558] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:20:10,742] INFO : Running validation on rank0 +rank0 [2026-10-28 11:20:10,742] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:20:10,742] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:20:11,823] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:20:11,966] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:20:12,095] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:20:12,169] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:20:12,237] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:20:12,304] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:20:12,373] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:20:12,441] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:20:12,544] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:20:12,667] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:20:12,789] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:20:12,861] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:20:12,930] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:20:12,998] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:20:13,067] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:20:13,136] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:20:13,248] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:20:13,255] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:20:13,258] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:20:13,261] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:20:13,263] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:20:13,265] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:20:13,268] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:20:13,270] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:20:13,271] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:20:13,274] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:20:13,277] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:20:13,279] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:20:13,282] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:20:13,284] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:20:13,286] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:20:13,289] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:20:19,018] INFO : VALIDATION | Step=2000/50000 | Train Loss=13.6430 | Valid Loss=14.0077 | Train Loss Unweighted=4.4829 | Valid Loss Unweighted=4.5018 | Stale=0 | ETA=186.2m +rank0 [2026-10-28 11:20:19,026] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:19,093] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:20:19,097] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:19,239] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:20,670] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:21,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:21,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:22,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:22,468] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:22,471] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:22,528] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:22,531] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:22,670] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:24,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:24,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:25,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:25,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:25,867] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:25,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:25,924] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:25,927] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:20:26,001] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:27,270] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:27,855] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:28,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:28,885] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:28,953] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:20:28,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:29,212] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:29,662] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:31,087] INFO : Plotted jets +rank0 [2026-10-28 11:20:32,164] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:32,197] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:32,422] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:32,795] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:34,583] INFO : Plotted jets +rank0 [2026-10-28 11:20:35,687] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:35,721] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:35,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:36,333] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:37,714] INFO : Plotted jets +rank0 [2026-10-28 11:20:38,802] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:57,150] INFO : Step 2100/50000 rank0 | Train Loss: 13.9090 | Train Loss Unweighted: 4.4638 | Train Loss Components: Classification: 0.0408 | Classification_binary: 3.9347 | Regression_cos_phi: 0.0123 | Regression_energy: 0.2244 | Regression_eta: 0.0132 | Regression_pt: 0.2250 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.3861 | Classification_binary: 3.9347 | Regression_cos_phi: 0.6698 | Regression_energy: 3.0209 | Regression_eta: 0.5551 | Regression_pt: 3.2908 | Regression_sin_phi: 1.0515 | LR: 9.96e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2202s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0340 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.3716 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 3010.8 | uss_MB: 2501.0 | child_rss_MB: 3248.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:20:57,366] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:15,235] INFO : Step 2200/50000 rank0 | Train Loss: 13.2996 | Train Loss Unweighted: 4.3393 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.8320 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2164 | Regression_eta: 0.0152 | Regression_pt: 0.2169 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1964 | Classification_binary: 3.8320 | Regression_cos_phi: 0.6959 | Regression_energy: 2.9132 | Regression_eta: 0.6360 | Regression_pt: 3.1722 | Regression_sin_phi: 0.8538 | LR: 9.96e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2057s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0626 | max_loss: 0.0031 | max_backward: 0.0384 | max_optimizer_step: 0.1033 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3010.8 | uss_MB: 2503.4 | child_rss_MB: 3246.6 | allocated_MB: 210.7 | reserved_MB: 2792.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:21:15,449] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:33,136] INFO : Step 2300/50000 rank0 | Train Loss: 13.9989 | Train Loss Unweighted: 4.5055 | Train Loss Components: Classification: 0.0379 | Classification_binary: 3.9602 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2362 | Regression_eta: 0.0131 | Regression_pt: 0.2344 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2861 | Classification_binary: 3.9602 | Regression_cos_phi: 0.6195 | Regression_energy: 3.1793 | Regression_eta: 0.5504 | Regression_pt: 3.4285 | Regression_sin_phi: 0.9748 | LR: 9.95e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0157 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.2054 | max_tensorboard_logging: 0.2590 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3010.8 | uss_MB: 2501.8 | child_rss_MB: 3246.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:21:33,360] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:50,958] INFO : Step 2400/50000 rank0 | Train Loss: 13.4525 | Train Loss Unweighted: 4.4140 | Train Loss Components: Classification: 0.0364 | Classification_binary: 3.9141 | Regression_cos_phi: 0.0124 | Regression_energy: 0.2130 | Regression_eta: 0.0150 | Regression_pt: 0.2097 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2375 | Classification_binary: 3.9141 | Regression_cos_phi: 0.6756 | Regression_energy: 2.8670 | Regression_eta: 0.6268 | Regression_pt: 3.0675 | Regression_sin_phi: 1.0641 | LR: 9.95e-04 | DataLoad Time: 0.0535s | Model Forward Time: 1.2218s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0857 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3604 | max_tensorboard_flush: 0.0133 | Memory: rss_MB: 3010.8 | uss_MB: 2502.2 | child_rss_MB: 3244.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:21:51,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:08,848] INFO : Step 2500/50000 rank0 | Train Loss: 13.2218 | Train Loss Unweighted: 4.4375 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.9428 | Regression_cos_phi: 0.0124 | Regression_energy: 0.2096 | Regression_eta: 0.0123 | Regression_pt: 0.2115 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2727 | Classification_binary: 3.9428 | Regression_cos_phi: 0.6727 | Regression_energy: 2.8215 | Regression_eta: 0.5153 | Regression_pt: 3.0934 | Regression_sin_phi: 0.9034 | LR: 9.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1525s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.1375 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2596 | max_tensorboard_flush: 0.0085 | Memory: rss_MB: 3010.8 | uss_MB: 2502.4 | child_rss_MB: 3242.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:22:09,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:26,376] INFO : Step 2600/50000 rank0 | Train Loss: 13.9761 | Train Loss Unweighted: 4.4610 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.9250 | Regression_cos_phi: 0.0144 | Regression_energy: 0.2290 | Regression_eta: 0.0142 | Regression_pt: 0.2317 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1249 | Classification_binary: 3.9250 | Regression_cos_phi: 0.7806 | Regression_energy: 3.0835 | Regression_eta: 0.5940 | Regression_pt: 3.3879 | Regression_sin_phi: 1.0804 | LR: 9.94e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0030 | max_backward: 0.1410 | max_optimizer_step: 0.1057 | max_tensorboard_logging: 0.2613 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 3010.8 | uss_MB: 2502.8 | child_rss_MB: 3240.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:22:26,604] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:43,959] INFO : Step 2700/50000 rank0 | Train Loss: 14.3597 | Train Loss Unweighted: 4.5058 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.9467 | Regression_cos_phi: 0.0145 | Regression_energy: 0.2388 | Regression_eta: 0.0140 | Regression_pt: 0.2414 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2757 | Classification_binary: 3.9467 | Regression_cos_phi: 0.7873 | Regression_energy: 3.2144 | Regression_eta: 0.5887 | Regression_pt: 3.5306 | Regression_sin_phi: 1.0163 | LR: 9.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.1675s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0148 | max_loss: 0.0029 | max_backward: 0.1167 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.2996 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2874.9 | uss_MB: 2366.1 | child_rss_MB: 3243.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:22:44,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:01,655] INFO : Step 2800/50000 rank0 | Train Loss: 13.2675 | Train Loss Unweighted: 4.3890 | Train Loss Components: Classification: 0.0363 | Classification_binary: 3.9086 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2006 | Regression_eta: 0.0170 | Regression_pt: 0.1999 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2324 | Classification_binary: 3.9086 | Regression_cos_phi: 0.7016 | Regression_energy: 2.7004 | Regression_eta: 0.7121 | Regression_pt: 2.9239 | Regression_sin_phi: 1.0885 | LR: 9.93e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2176s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0145 | max_loss: 0.0409 | max_backward: 0.0336 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2874.9 | uss_MB: 2365.5 | child_rss_MB: 3266.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:23:01,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:19,237] INFO : Step 2900/50000 rank0 | Train Loss: 13.2001 | Train Loss Unweighted: 4.2595 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7678 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2086 | Regression_eta: 0.0160 | Regression_pt: 0.2055 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2450 | Classification_binary: 3.7678 | Regression_cos_phi: 0.6078 | Regression_energy: 2.8076 | Regression_eta: 0.6689 | Regression_pt: 3.0049 | Regression_sin_phi: 1.0982 | LR: 9.93e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.1964s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0503 | max_loss: 0.0029 | max_backward: 0.0335 | max_optimizer_step: 0.1000 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 2875.9 | uss_MB: 2368.2 | child_rss_MB: 3238.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:23:19,452] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:36,846] INFO : Step 3000/50000 rank0 | Train Loss: 12.9697 | Train Loss Unweighted: 4.3787 | Train Loss Components: Classification: 0.0363 | Classification_binary: 3.9002 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2025 | Regression_eta: 0.0130 | Regression_pt: 0.2024 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.2330 | Classification_binary: 3.9002 | Regression_cos_phi: 0.6999 | Regression_energy: 2.7262 | Regression_eta: 0.5441 | Regression_pt: 2.9601 | Regression_sin_phi: 0.9062 | LR: 9.92e-04 | DataLoad Time: 0.1146s | Model Forward Time: 1.2229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0457 | max_forward: 0.0156 | max_loss: 0.0029 | max_backward: 0.0333 | max_optimizer_step: 0.1014 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2875.9 | uss_MB: 2368.7 | child_rss_MB: 3238.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:23:37,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:37,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:23:37,231] INFO : Running validation on rank0 +rank0 [2026-10-28 11:23:37,231] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:23:37,231] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:23:38,304] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:23:38,446] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:23:38,576] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:23:38,649] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:23:38,717] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:23:38,783] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:23:38,852] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:23:38,920] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:23:39,024] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:23:39,145] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:23:39,266] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:23:39,337] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:23:39,405] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:23:39,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:23:39,541] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:23:39,610] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:23:39,721] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:23:39,728] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:23:39,730] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:23:39,733] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:23:39,736] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:23:39,737] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:23:39,740] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:23:39,742] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:23:39,743] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:23:39,746] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:23:39,748] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:23:39,749] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:23:39,752] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:23:39,755] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:23:39,757] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:23:39,758] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:23:45,623] INFO : VALIDATION | Step=3000/50000 | Train Loss=12.9697 | Valid Loss=13.2191 | Train Loss Unweighted=4.3787 | Valid Loss Unweighted=4.3149 | Stale=0 | ETA=175.5m +rank0 [2026-10-28 11:23:45,633] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:45,693] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:23:45,697] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:45,839] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:47,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:48,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:48,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:49,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:49,243] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:23:49,247] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:49,302] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:23:49,304] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:49,380] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:50,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:51,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:51,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:52,552] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:52,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:52,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:23:52,686] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:23:52,689] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:23:52,761] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:23:54,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:23:54,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:23:55,180] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:23:55,679] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:23:55,751] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:23:55,759] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:56,005] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:23:56,461] INFO : Plotted number of elements +rank0 [2026-10-28 11:23:57,889] INFO : Plotted jets +rank0 [2026-10-28 11:23:58,970] INFO : Plotted jet ratio +rank0 [2026-10-28 11:23:59,004] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:59,230] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:23:59,603] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:01,358] INFO : Plotted jets +rank0 [2026-10-28 11:24:02,470] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:02,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:24:02,702] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:24:03,116] INFO : Plotted number of elements +rank0 [2026-10-28 11:24:04,496] INFO : Plotted jets +rank0 [2026-10-28 11:24:05,589] INFO : Plotted jet ratio +rank0 [2026-10-28 11:24:23,304] INFO : Step 3100/50000 rank0 | Train Loss: 13.5068 | Train Loss Unweighted: 4.3736 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.8624 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2195 | Regression_eta: 0.0127 | Regression_pt: 0.2179 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1654 | Classification_binary: 3.8624 | Regression_cos_phi: 0.6915 | Regression_energy: 2.9554 | Regression_eta: 0.5329 | Regression_pt: 3.1873 | Regression_sin_phi: 1.1120 | LR: 9.91e-04 | DataLoad Time: 0.1919s | Model Forward Time: 1.0768s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.1031 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3089.1 | uss_MB: 2580.5 | child_rss_MB: 3237.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:24:23,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:40,840] INFO : Step 3200/50000 rank0 | Train Loss: 12.9836 | Train Loss Unweighted: 4.3833 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.9012 | Regression_cos_phi: 0.0115 | Regression_energy: 0.2042 | Regression_eta: 0.0154 | Regression_pt: 0.2058 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1402 | Classification_binary: 3.9012 | Regression_cos_phi: 0.6238 | Regression_energy: 2.7494 | Regression_eta: 0.6460 | Regression_pt: 3.0101 | Regression_sin_phi: 0.9128 | LR: 9.91e-04 | DataLoad Time: 0.1708s | Model Forward Time: 1.1519s | Phase Times: max_pre_step_barrier: 0.0119 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1058 | max_tensorboard_logging: 0.3147 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3089.1 | uss_MB: 2580.8 | child_rss_MB: 3236.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:24:41,056] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:58,359] INFO : Step 3300/50000 rank0 | Train Loss: 13.2314 | Train Loss Unweighted: 4.2246 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.7231 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2121 | Regression_eta: 0.0141 | Regression_pt: 0.2151 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1837 | Classification_binary: 3.7231 | Regression_cos_phi: 0.6084 | Regression_energy: 2.8552 | Regression_eta: 0.5891 | Regression_pt: 3.1455 | Regression_sin_phi: 1.1264 | LR: 9.90e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.0888s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0031 | max_backward: 0.0331 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3089.1 | uss_MB: 2581.1 | child_rss_MB: 3234.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:24:58,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:15,901] INFO : Step 3400/50000 rank0 | Train Loss: 12.4928 | Train Loss Unweighted: 4.1350 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.6646 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1999 | Regression_eta: 0.0121 | Regression_pt: 0.2027 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0997 | Classification_binary: 3.6646 | Regression_cos_phi: 0.6160 | Regression_energy: 2.6907 | Regression_eta: 0.5090 | Regression_pt: 2.9646 | Regression_sin_phi: 0.9481 | LR: 9.90e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.0837s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3089.1 | uss_MB: 2581.4 | child_rss_MB: 3233.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:25:16,117] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:33,436] INFO : Step 3500/50000 rank0 | Train Loss: 12.7254 | Train Loss Unweighted: 4.3004 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.8316 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1973 | Regression_eta: 0.0142 | Regression_pt: 0.2003 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1242 | Classification_binary: 3.8316 | Regression_cos_phi: 0.6718 | Regression_energy: 2.6556 | Regression_eta: 0.5968 | Regression_pt: 2.9298 | Regression_sin_phi: 0.9156 | LR: 9.89e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0165 | max_loss: 0.0031 | max_backward: 0.0374 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3089.1 | uss_MB: 2583.7 | child_rss_MB: 3233.7 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:25:33,652] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:50,865] INFO : Step 3600/50000 rank0 | Train Loss: 11.4558 | Train Loss Unweighted: 3.9913 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.5610 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1845 | Regression_eta: 0.0094 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1008 | Classification_binary: 3.5610 | Regression_cos_phi: 0.5221 | Regression_energy: 2.4843 | Regression_eta: 0.3944 | Regression_pt: 2.7168 | Regression_sin_phi: 0.6764 | LR: 9.89e-04 | DataLoad Time: 0.0885s | Model Forward Time: 1.3003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1046 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3089.1 | uss_MB: 2581.7 | child_rss_MB: 3228.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:25:51,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:08,266] INFO : Step 3700/50000 rank0 | Train Loss: 11.7007 | Train Loss Unweighted: 4.1144 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.6838 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1846 | Regression_eta: 0.0103 | Regression_pt: 0.1836 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1114 | Classification_binary: 3.6838 | Regression_cos_phi: 0.5157 | Regression_energy: 2.4852 | Regression_eta: 0.4335 | Regression_pt: 2.6848 | Regression_sin_phi: 0.7863 | LR: 9.88e-04 | DataLoad Time: 0.0533s | Model Forward Time: 1.2210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0565 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1051 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2953.1 | uss_MB: 2445.8 | child_rss_MB: 3233.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:26:08,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:25,777] INFO : Step 3800/50000 rank0 | Train Loss: 12.2407 | Train Loss Unweighted: 4.1208 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.6893 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1804 | Regression_eta: 0.0153 | Regression_pt: 0.1781 | Regression_sin_phi: 0.0155 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0763 | Classification_binary: 3.6893 | Regression_cos_phi: 0.5752 | Regression_energy: 2.4289 | Regression_eta: 0.6416 | Regression_pt: 2.6044 | Regression_sin_phi: 1.2252 | LR: 9.87e-04 | DataLoad Time: 0.1080s | Model Forward Time: 1.2511s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0893 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1042 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 2953.1 | uss_MB: 2446.4 | child_rss_MB: 3232.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:26:25,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:43,285] INFO : Step 3900/50000 rank0 | Train Loss: 13.7422 | Train Loss Unweighted: 4.0877 | Train Loss Components: Classification: 0.0348 | Classification_binary: 3.5466 | Regression_cos_phi: 0.0111 | Regression_energy: 0.2271 | Regression_eta: 0.0269 | Regression_pt: 0.2304 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1826 | Classification_binary: 3.5466 | Regression_cos_phi: 0.6043 | Regression_energy: 3.0574 | Regression_eta: 1.1283 | Regression_pt: 3.3695 | Regression_sin_phi: 0.8536 | LR: 9.87e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0532 | max_loss: 0.0032 | max_backward: 0.0384 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 2953.1 | uss_MB: 2449.3 | child_rss_MB: 3234.1 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 11:26:43,499] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:00,788] INFO : Step 4000/50000 rank0 | Train Loss: 13.3953 | Train Loss Unweighted: 4.2353 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7303 | Regression_cos_phi: 0.0133 | Regression_energy: 0.2176 | Regression_eta: 0.0185 | Regression_pt: 0.2079 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1787 | Classification_binary: 3.7303 | Regression_cos_phi: 0.7229 | Regression_energy: 2.9300 | Regression_eta: 0.7735 | Regression_pt: 3.0411 | Regression_sin_phi: 1.0188 | LR: 9.86e-04 | DataLoad Time: 0.0335s | Model Forward Time: 1.2911s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1222 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 2954.1 | uss_MB: 2447.8 | child_rss_MB: 3234.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:01,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:27:01,021] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:27:01,208] INFO : Running validation on rank0 +rank0 [2026-10-28 11:27:01,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:27:01,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:02,268] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:27:02,410] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:02,541] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:02,614] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:02,682] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:02,749] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:02,819] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:02,888] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:02,996] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:03,117] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:03,243] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:03,313] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:03,381] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:03,448] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:03,517] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:03,586] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:03,702] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:03,710] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:03,712] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:03,715] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:03,716] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:03,719] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:03,721] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:03,724] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:03,725] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:03,728] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:03,728] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:03,729] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:03,732] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:03,735] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:03,736] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:03,738] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:09,538] INFO : VALIDATION | Step=4000/50000 | Train Loss=13.3953 | Valid Loss=12.3813 | Train Loss Unweighted=4.2353 | Valid Loss Unweighted=4.1399 | Stale=0 | ETA=167.9m +rank0 [2026-10-28 11:27:09,547] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:09,609] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:09,612] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:09,758] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:11,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:11,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:12,256] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:12,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:12,945] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:12,947] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:13,003] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:13,006] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:13,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:14,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:15,014] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:15,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:16,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:16,231] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:16,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:16,288] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:16,290] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:27:16,368] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:17,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:18,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:18,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:19,413] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:19,486] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:27:19,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:19,742] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:20,193] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:21,620] INFO : Plotted jets +rank0 [2026-10-28 11:27:22,702] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:22,735] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:22,956] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:23,348] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:25,136] INFO : Plotted jets +rank0 [2026-10-28 11:27:26,247] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:26,279] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:26,480] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:26,892] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:28,275] INFO : Plotted jets +rank0 [2026-10-28 11:27:29,369] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:46,648] INFO : Step 4100/50000 rank0 | Train Loss: 12.0936 | Train Loss Unweighted: 4.1041 | Train Loss Components: Classification: 0.0330 | Classification_binary: 3.6632 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1863 | Regression_eta: 0.0137 | Regression_pt: 0.1851 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1200 | Classification_binary: 3.6632 | Regression_cos_phi: 0.6062 | Regression_energy: 2.5087 | Regression_eta: 0.5755 | Regression_pt: 2.7072 | Regression_sin_phi: 0.9128 | LR: 9.85e-04 | DataLoad Time: 0.0292s | Model Forward Time: 1.1405s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1657 | max_tensorboard_logging: 0.2765 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3125.9 | uss_MB: 2621.2 | child_rss_MB: 3232.0 | allocated_MB: 207.0 | reserved_MB: 2620.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:27:46,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:03,923] INFO : Step 4200/50000 rank0 | Train Loss: 11.6792 | Train Loss Unweighted: 4.1039 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.6639 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1896 | Regression_eta: 0.0104 | Regression_pt: 0.1900 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1176 | Classification_binary: 3.6639 | Regression_cos_phi: 0.4762 | Regression_energy: 2.5530 | Regression_eta: 0.4342 | Regression_pt: 2.7786 | Regression_sin_phi: 0.6556 | LR: 9.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1032s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3126.9 | uss_MB: 2621.5 | child_rss_MB: 3230.1 | allocated_MB: 207.0 | reserved_MB: 2620.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:04,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:20,968] INFO : Step 4300/50000 rank0 | Train Loss: 11.3414 | Train Loss Unweighted: 3.8949 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.4934 | Regression_cos_phi: 0.0121 | Regression_energy: 0.1687 | Regression_eta: 0.0126 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0356 | Classification_binary: 3.4934 | Regression_cos_phi: 0.6588 | Regression_energy: 2.2711 | Regression_eta: 0.5283 | Regression_pt: 2.4247 | Regression_sin_phi: 0.9295 | LR: 9.84e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.1365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0332 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3126.9 | uss_MB: 2621.9 | child_rss_MB: 3227.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:28:21,192] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:38,171] INFO : Step 4400/50000 rank0 | Train Loss: 11.1695 | Train Loss Unweighted: 4.0353 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.6355 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1709 | Regression_eta: 0.0119 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0032 | Classification_binary: 3.6355 | Regression_cos_phi: 0.5771 | Regression_energy: 2.3007 | Regression_eta: 0.4990 | Regression_pt: 2.4582 | Regression_sin_phi: 0.6958 | LR: 9.83e-04 | DataLoad Time: 0.1882s | Model Forward Time: 1.1807s | Phase Times: max_pre_step_barrier: 0.0684 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1027 | max_tensorboard_logging: 0.3059 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3126.9 | uss_MB: 2622.2 | child_rss_MB: 3226.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:38,384] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:55,260] INFO : Step 4500/50000 rank0 | Train Loss: 12.5243 | Train Loss Unweighted: 4.1024 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.6156 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2098 | Regression_eta: 0.0112 | Regression_pt: 0.2110 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1611 | Classification_binary: 3.6156 | Regression_cos_phi: 0.5788 | Regression_energy: 2.8249 | Regression_eta: 0.4714 | Regression_pt: 3.0859 | Regression_sin_phi: 0.7867 | LR: 9.82e-04 | DataLoad Time: 0.0149s | Model Forward Time: 1.1967s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.2842 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3126.9 | uss_MB: 2622.4 | child_rss_MB: 3225.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:28:55,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:12,254] INFO : Step 4600/50000 rank0 | Train Loss: 13.1041 | Train Loss Unweighted: 4.1161 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.6132 | Regression_cos_phi: 0.0120 | Regression_energy: 0.2134 | Regression_eta: 0.0141 | Regression_pt: 0.2164 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1336 | Classification_binary: 3.6132 | Regression_cos_phi: 0.6547 | Regression_energy: 2.8726 | Regression_eta: 0.5922 | Regression_pt: 3.1648 | Regression_sin_phi: 1.0730 | LR: 9.81e-04 | DataLoad Time: 0.0899s | Model Forward Time: 1.2683s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.0175 | max_loss: 0.0866 | max_backward: 0.0380 | max_optimizer_step: 0.1037 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3127.9 | uss_MB: 2623.6 | child_rss_MB: 3224.7 | allocated_MB: 210.7 | reserved_MB: 2740.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:29:12,469] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:29,298] INFO : Step 4700/50000 rank0 | Train Loss: 11.6841 | Train Loss Unweighted: 3.9829 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.5536 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1834 | Regression_eta: 0.0119 | Regression_pt: 0.1811 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0671 | Classification_binary: 3.5536 | Regression_cos_phi: 0.5649 | Regression_energy: 2.4691 | Regression_eta: 0.4994 | Regression_pt: 2.6481 | Regression_sin_phi: 0.8819 | LR: 9.81e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.0895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1011 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3027.8 | uss_MB: 2523.0 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:29,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:46,072] INFO : Step 4800/50000 rank0 | Train Loss: 11.3393 | Train Loss Unweighted: 3.9175 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.5087 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1710 | Regression_eta: 0.0120 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0445 | Classification_binary: 3.5087 | Regression_cos_phi: 0.6225 | Regression_energy: 2.3015 | Regression_eta: 0.5011 | Regression_pt: 2.5317 | Regression_sin_phi: 0.8293 | LR: 9.80e-04 | DataLoad Time: 0.0099s | Model Forward Time: 1.0692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0132 | max_loss: 0.0029 | max_backward: 0.0329 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3029.8 | uss_MB: 2525.0 | child_rss_MB: 3223.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:29:46,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:03,452] INFO : Step 4900/50000 rank0 | Train Loss: 11.1474 | Train Loss Unweighted: 3.8611 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.4561 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1717 | Regression_eta: 0.0105 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0501 | Classification_binary: 3.4561 | Regression_cos_phi: 0.5447 | Regression_energy: 2.3114 | Regression_eta: 0.4396 | Regression_pt: 2.5045 | Regression_sin_phi: 0.8410 | LR: 9.79e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2750s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1131 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1039 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3029.8 | uss_MB: 2525.5 | child_rss_MB: 3219.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:03,665] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:20,636] INFO : Step 5000/50000 rank0 | Train Loss: 12.4812 | Train Loss Unweighted: 4.0496 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.5790 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2003 | Regression_eta: 0.0164 | Regression_pt: 0.1994 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0789 | Classification_binary: 3.5790 | Regression_cos_phi: 0.6195 | Regression_energy: 2.6961 | Regression_eta: 0.6876 | Regression_pt: 2.9157 | Regression_sin_phi: 0.9045 | LR: 9.78e-04 | DataLoad Time: 0.0624s | Model Forward Time: 1.1579s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0029 | max_backward: 0.0328 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3029.8 | uss_MB: 2525.9 | child_rss_MB: 3221.2 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:30:20,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:20,880] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:30:20,962] INFO : Running validation on rank0 +rank0 [2026-10-28 11:30:20,962] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:30:20,962] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:30:22,145] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:30:22,288] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:30:22,418] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:30:22,491] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:30:22,558] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:30:22,625] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:30:22,694] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:30:22,762] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:30:22,871] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:30:22,995] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:30:23,116] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:30:23,187] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:30:23,256] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:30:23,323] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:30:23,392] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:30:23,460] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:30:23,573] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:30:23,581] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:30:23,583] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:30:23,586] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:30:23,587] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:30:23,590] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:30:23,593] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:30:23,594] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:30:23,596] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:30:23,599] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:30:23,600] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:30:23,602] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:30:23,603] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:30:23,606] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:30:23,607] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:30:23,610] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:30:29,498] INFO : VALIDATION | Step=5000/50000 | Train Loss=12.4812 | Valid Loss=11.7654 | Train Loss Unweighted=4.0496 | Valid Loss Unweighted=3.9604 | Stale=0 | ETA=161.4m +rank0 [2026-10-28 11:30:29,507] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:29,570] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:30:29,574] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:29,703] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:31,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:31,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:32,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:32,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:32,937] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:32,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:32,996] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:30:32,999] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:33,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:34,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:34,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:35,482] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:36,008] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:36,085] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:36,089] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:30:36,142] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:30:36,145] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:30:36,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:30:37,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:30:38,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:30:38,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:30:39,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:30:39,216] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:30:39,223] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:39,471] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:30:39,924] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:41,354] INFO : Plotted jets +rank0 [2026-10-28 11:30:42,433] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:42,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:42,691] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:30:43,064] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:44,831] INFO : Plotted jets +rank0 [2026-10-28 11:30:45,942] INFO : Plotted jet ratio +rank0 [2026-10-28 11:30:45,975] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:30:46,175] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:30:46,586] INFO : Plotted number of elements +rank0 [2026-10-28 11:30:47,965] INFO : Plotted jets +rank0 [2026-10-28 11:30:49,050] INFO : Plotted jet ratio +rank0 [2026-10-28 11:31:06,102] INFO : Step 5100/50000 rank0 | Train Loss: 11.7930 | Train Loss Unweighted: 3.8677 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.4254 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1892 | Regression_eta: 0.0113 | Regression_pt: 0.1888 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9747 | Classification_binary: 3.4254 | Regression_cos_phi: 0.6876 | Regression_energy: 2.5471 | Regression_eta: 0.4726 | Regression_pt: 2.7613 | Regression_sin_phi: 0.9243 | LR: 9.77e-04 | DataLoad Time: 0.0134s | Model Forward Time: 1.1108s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1087 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3098.9 | uss_MB: 2598.3 | child_rss_MB: 3224.5 | allocated_MB: 207.0 | reserved_MB: 2600.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:06,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:22,854] INFO : Step 5200/50000 rank0 | Train Loss: 11.8786 | Train Loss Unweighted: 3.9343 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.4876 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1910 | Regression_eta: 0.0121 | Regression_pt: 0.1903 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0691 | Classification_binary: 3.4876 | Regression_cos_phi: 0.5862 | Regression_energy: 2.5708 | Regression_eta: 0.5071 | Regression_pt: 2.7826 | Regression_sin_phi: 0.8752 | LR: 9.76e-04 | DataLoad Time: 0.0397s | Model Forward Time: 1.1914s | Phase Times: max_pre_step_barrier: 0.0680 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1054 | max_tensorboard_logging: 0.3068 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3100.9 | uss_MB: 2600.1 | child_rss_MB: 3223.5 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:31:23,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:39,647] INFO : Step 5300/50000 rank0 | Train Loss: 10.6609 | Train Loss Unweighted: 3.8564 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.4696 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1660 | Regression_eta: 0.0102 | Regression_pt: 0.1656 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9370 | Classification_binary: 3.4696 | Regression_cos_phi: 0.4540 | Regression_energy: 2.2341 | Regression_eta: 0.4273 | Regression_pt: 2.4223 | Regression_sin_phi: 0.7166 | LR: 9.75e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.0593s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0030 | max_backward: 0.0325 | max_optimizer_step: 0.1017 | max_tensorboard_logging: 0.2745 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3100.9 | uss_MB: 2600.6 | child_rss_MB: 3223.9 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:31:39,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:56,481] INFO : Step 5400/50000 rank0 | Train Loss: 11.6733 | Train Loss Unweighted: 3.9370 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.5066 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1838 | Regression_eta: 0.0129 | Regression_pt: 0.1812 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0224 | Classification_binary: 3.5066 | Regression_cos_phi: 0.6339 | Regression_energy: 2.4741 | Regression_eta: 0.5415 | Regression_pt: 2.6499 | Regression_sin_phi: 0.8450 | LR: 9.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.1020s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1072 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3100.9 | uss_MB: 2601.3 | child_rss_MB: 3221.9 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:31:56,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:13,173] INFO : Step 5500/50000 rank0 | Train Loss: 11.6202 | Train Loss Unweighted: 3.7524 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.3283 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1758 | Regression_eta: 0.0188 | Regression_pt: 0.1754 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.0647 | Classification_binary: 3.3283 | Regression_cos_phi: 0.6411 | Regression_energy: 2.3669 | Regression_eta: 0.7863 | Regression_pt: 2.5655 | Regression_sin_phi: 0.8676 | LR: 9.73e-04 | DataLoad Time: 0.0211s | Model Forward Time: 1.2304s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0520 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.3739 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3100.9 | uss_MB: 2601.5 | child_rss_MB: 3224.1 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:32:13,385] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:29,921] INFO : Step 5600/50000 rank0 | Train Loss: 11.1195 | Train Loss Unweighted: 3.8400 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.4180 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1838 | Regression_eta: 0.0118 | Regression_pt: 0.1823 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8854 | Classification_binary: 3.4180 | Regression_cos_phi: 0.5596 | Regression_energy: 2.4741 | Regression_eta: 0.4957 | Regression_pt: 2.6654 | Regression_sin_phi: 0.6213 | LR: 9.72e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2505s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0872 | max_loss: 0.0031 | max_backward: 0.0342 | max_optimizer_step: 0.1035 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3100.9 | uss_MB: 2601.7 | child_rss_MB: 3221.3 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:32:30,134] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:47,036] INFO : Step 5700/50000 rank0 | Train Loss: 11.1942 | Train Loss Unweighted: 3.8053 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3874 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1814 | Regression_eta: 0.0139 | Regression_pt: 0.1760 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9185 | Classification_binary: 3.3874 | Regression_cos_phi: 0.5778 | Regression_energy: 2.4424 | Regression_eta: 0.5820 | Regression_pt: 2.5734 | Regression_sin_phi: 0.7126 | LR: 9.71e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2230s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.2831 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3099.1 | uss_MB: 2599.1 | child_rss_MB: 3220.0 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:32:47,263] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 119.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:04,165] INFO : Step 5800/50000 rank0 | Train Loss: 11.4069 | Train Loss Unweighted: 3.7753 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.3449 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1869 | Regression_eta: 0.0111 | Regression_pt: 0.1835 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9090 | Classification_binary: 3.3449 | Regression_cos_phi: 0.5895 | Regression_energy: 2.5159 | Regression_eta: 0.4634 | Regression_pt: 2.6830 | Regression_sin_phi: 0.9011 | LR: 9.70e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1055 | max_tensorboard_logging: 0.2817 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3101.1 | uss_MB: 2601.1 | child_rss_MB: 3219.7 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:33:04,376] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:33:20,603] INFO : Step 5900/50000 rank0 | Train Loss: 11.5323 | Train Loss Unweighted: 3.7381 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.3077 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1820 | Regression_eta: 0.0130 | Regression_pt: 0.1805 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 1.1055 | Classification_binary: 3.3077 | Regression_cos_phi: 0.6180 | Regression_energy: 2.4508 | Regression_eta: 0.5432 | Regression_pt: 2.6391 | Regression_sin_phi: 0.8680 | LR: 9.69e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1670s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0915 | max_backward: 0.0342 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3101.1 | uss_MB: 2601.4 | child_rss_MB: 3244.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:33:20,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:37,279] INFO : Step 6000/50000 rank0 | Train Loss: 11.3068 | Train Loss Unweighted: 3.7757 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.3496 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1832 | Regression_eta: 0.0099 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 3.3496 | Regression_cos_phi: 0.6457 | Regression_energy: 2.4666 | Regression_eta: 0.4157 | Regression_pt: 2.6638 | Regression_sin_phi: 0.7739 | LR: 9.68e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1259s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0031 | max_backward: 0.0710 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3102.1 | uss_MB: 2602.1 | child_rss_MB: 3217.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:33:37,489] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:37,508] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:33:37,589] INFO : Running validation on rank0 +rank0 [2026-10-28 11:33:37,589] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:33:37,589] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:33:38,759] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:33:38,902] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:33:39,032] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:33:39,105] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:33:39,172] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:33:39,239] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:33:39,307] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:33:39,375] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:33:39,485] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:33:39,609] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:33:39,730] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:33:39,802] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:33:39,870] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:33:39,937] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:33:40,006] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:33:40,074] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:33:40,193] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:33:40,201] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:33:40,203] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:33:40,205] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:33:40,207] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:33:40,210] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:33:40,211] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:33:40,214] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:33:40,216] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:33:40,218] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:33:40,220] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:33:40,221] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:33:40,224] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:33:40,226] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:33:40,229] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:33:40,230] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:33:45,982] INFO : VALIDATION | Step=6000/50000 | Train Loss=11.3068 | Valid Loss=11.3637 | Train Loss Unweighted=3.7757 | Valid Loss Unweighted=3.8073 | Stale=0 | ETA=155.5m +rank0 [2026-10-28 11:33:45,991] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:46,052] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:33:46,056] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:46,202] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:47,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:48,087] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:48,658] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:49,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:49,289] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:49,293] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:49,349] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:33:49,352] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:33:49,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:50,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:51,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:51,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:52,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:52,625] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:52,628] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:33:52,683] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:33:52,686] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:33:52,747] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:33:54,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:33:54,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:33:55,184] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:33:55,682] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:33:55,761] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:33:55,768] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:56,020] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:33:56,472] INFO : Plotted number of elements +rank0 [2026-10-28 11:33:57,903] INFO : Plotted jets +rank0 [2026-10-28 11:33:58,986] INFO : Plotted jet ratio +rank0 [2026-10-28 11:33:59,017] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:33:59,240] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:00,038] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:01,424] INFO : Plotted jets +rank0 [2026-10-28 11:34:02,535] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:02,567] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:02,769] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:03,182] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:04,577] INFO : Plotted jets +rank0 [2026-10-28 11:34:05,671] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:22,354] INFO : Step 6100/50000 rank0 | Train Loss: 11.0077 | Train Loss Unweighted: 3.7377 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.3377 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1681 | Regression_eta: 0.0111 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9584 | Classification_binary: 3.3377 | Regression_cos_phi: 0.5566 | Regression_energy: 2.2633 | Regression_eta: 0.4664 | Regression_pt: 2.4945 | Regression_sin_phi: 0.9308 | LR: 9.67e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0857 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3041 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3174.6 | uss_MB: 2676.3 | child_rss_MB: 3218.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:22,568] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:38,827] INFO : Step 6200/50000 rank0 | Train Loss: 11.3747 | Train Loss Unweighted: 3.8074 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.3903 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1770 | Regression_eta: 0.0133 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9985 | Classification_binary: 3.3903 | Regression_cos_phi: 0.5511 | Regression_energy: 2.3830 | Regression_eta: 0.5582 | Regression_pt: 2.5675 | Regression_sin_phi: 0.9261 | LR: 9.66e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1046s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0029 | max_backward: 0.0342 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2860 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3175.6 | uss_MB: 2678.1 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:39,043] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:55,486] INFO : Step 6300/50000 rank0 | Train Loss: 10.3960 | Train Loss Unweighted: 3.7173 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.3363 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1626 | Regression_eta: 0.0091 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8930 | Classification_binary: 3.3363 | Regression_cos_phi: 0.4672 | Regression_energy: 2.1891 | Regression_eta: 0.3813 | Regression_pt: 2.4205 | Regression_sin_phi: 0.7086 | LR: 9.65e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.3216s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1119 | max_loss: 0.0031 | max_backward: 0.0344 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3895 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3175.6 | uss_MB: 2678.3 | child_rss_MB: 3217.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:34:55,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:11,928] INFO : Step 6400/50000 rank0 | Train Loss: 11.2701 | Train Loss Unweighted: 3.7425 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.3212 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1801 | Regression_eta: 0.0133 | Regression_pt: 0.1783 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9750 | Classification_binary: 3.3212 | Regression_cos_phi: 0.5876 | Regression_energy: 2.4240 | Regression_eta: 0.5591 | Regression_pt: 2.6080 | Regression_sin_phi: 0.7951 | LR: 9.64e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1983s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0029 | max_backward: 0.0337 | max_optimizer_step: 0.1780 | max_tensorboard_logging: 0.2942 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 3175.6 | uss_MB: 2678.6 | child_rss_MB: 3217.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:35:12,155] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:28,676] INFO : Step 6500/50000 rank0 | Train Loss: 11.3810 | Train Loss Unweighted: 3.7540 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.3339 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1786 | Regression_eta: 0.0134 | Regression_pt: 0.1774 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9364 | Classification_binary: 3.3339 | Regression_cos_phi: 0.6222 | Regression_energy: 2.4039 | Regression_eta: 0.5623 | Regression_pt: 2.5943 | Regression_sin_phi: 0.9281 | LR: 9.63e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1866 | max_tensorboard_logging: 0.3098 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3175.6 | uss_MB: 2678.7 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:35:28,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:35:45,191] INFO : Step 6600/50000 rank0 | Train Loss: 10.7826 | Train Loss Unweighted: 3.7040 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.3184 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1621 | Regression_eta: 0.0113 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9654 | Classification_binary: 3.3184 | Regression_cos_phi: 0.5792 | Regression_energy: 2.1826 | Regression_eta: 0.4721 | Regression_pt: 2.3658 | Regression_sin_phi: 0.8992 | LR: 9.62e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2929s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0897 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1088 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.6 | uss_MB: 2678.9 | child_rss_MB: 3212.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:35:45,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:01,966] INFO : Step 6700/50000 rank0 | Train Loss: 10.9594 | Train Loss Unweighted: 3.6751 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.2768 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1681 | Regression_eta: 0.0131 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9101 | Classification_binary: 3.2768 | Regression_cos_phi: 0.6282 | Regression_energy: 2.2629 | Regression_eta: 0.5509 | Regression_pt: 2.4506 | Regression_sin_phi: 0.8799 | LR: 9.61e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1029s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0030 | max_backward: 0.0337 | max_optimizer_step: 0.1052 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3176.6 | uss_MB: 2679.2 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:02,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:18,634] INFO : Step 6800/50000 rank0 | Train Loss: 10.6312 | Train Loss Unweighted: 3.6811 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.2920 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1672 | Regression_eta: 0.0120 | Regression_pt: 0.1642 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8820 | Classification_binary: 3.2920 | Regression_cos_phi: 0.5566 | Regression_energy: 2.2515 | Regression_eta: 0.5019 | Regression_pt: 2.4011 | Regression_sin_phi: 0.7460 | LR: 9.60e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2216s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0425 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1071 | max_tensorboard_logging: 0.3780 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3176.9 | uss_MB: 2678.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:36:18,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:34,723] INFO : Step 6900/50000 rank0 | Train Loss: 10.4078 | Train Loss Unweighted: 3.5532 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.1589 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1720 | Regression_eta: 0.0084 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8811 | Classification_binary: 3.1589 | Regression_cos_phi: 0.5015 | Regression_energy: 2.3162 | Regression_eta: 0.3541 | Regression_pt: 2.4806 | Regression_sin_phi: 0.7153 | LR: 9.58e-04 | DataLoad Time: 0.1567s | Model Forward Time: 1.3090s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1199 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1041 | max_tensorboard_logging: 0.3822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.9 | uss_MB: 2679.0 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:34,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:51,259] INFO : Step 7000/50000 rank0 | Train Loss: 11.0857 | Train Loss Unweighted: 3.6867 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.2770 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1745 | Regression_eta: 0.0124 | Regression_pt: 0.1736 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9206 | Classification_binary: 3.2770 | Regression_cos_phi: 0.5937 | Regression_energy: 2.3498 | Regression_eta: 0.5189 | Regression_pt: 2.5382 | Regression_sin_phi: 0.8875 | LR: 9.57e-04 | DataLoad Time: 0.0868s | Model Forward Time: 1.2075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.1285 | max_optimizer_step: 0.1064 | max_tensorboard_logging: 0.2866 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3176.9 | uss_MB: 2679.2 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:36:51,475] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:51,493] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 11:36:51,573] INFO : Running validation on rank0 +rank0 [2026-10-28 11:36:51,574] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:36:51,574] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:36:52,733] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 11:36:52,877] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:36:53,008] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:36:53,081] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:36:53,148] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:36:53,217] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:36:53,285] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:36:53,353] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:36:53,463] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:36:53,585] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:36:53,706] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:36:53,777] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:36:53,845] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:36:53,913] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:36:53,982] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:36:54,050] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:36:54,154] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:36:54,161] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:36:54,164] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:36:54,167] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:36:54,168] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:36:54,170] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:36:54,173] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:36:54,175] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:36:54,177] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:36:54,179] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:36:54,182] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:36:54,183] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:36:54,185] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:36:54,186] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:36:54,189] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:36:54,191] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:36:59,974] INFO : VALIDATION | Step=7000/50000 | Train Loss=11.0857 | Valid Loss=10.8608 | Train Loss Unweighted=3.6867 | Valid Loss Unweighted=3.6578 | Stale=0 | ETA=150.1m +rank0 [2026-10-28 11:36:59,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:00,046] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:37:00,049] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:00,198] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:01,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:02,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:02,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:03,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:03,642] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:37:03,645] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:03,701] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:37:03,704] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:03,759] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:05,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:05,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:06,324] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:06,859] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:06,948] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:06,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:37:07,005] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:37:07,008] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:37:07,088] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:37:08,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:37:08,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:37:09,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:37:10,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:37:10,089] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:37:10,097] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:10,346] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:37:10,794] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:12,228] INFO : Plotted jets +rank0 [2026-10-28 11:37:13,308] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:13,340] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:13,563] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:37:13,941] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:15,711] INFO : Plotted jets +rank0 [2026-10-28 11:37:16,818] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:16,851] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 11:37:17,052] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 11:37:17,478] INFO : Plotted number of elements +rank0 [2026-10-28 11:37:18,863] INFO : Plotted jets +rank0 [2026-10-28 11:37:19,953] INFO : Plotted jet ratio +rank0 [2026-10-28 11:37:36,286] INFO : Step 7100/50000 rank0 | Train Loss: 10.7771 | Train Loss Unweighted: 3.6153 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2144 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1711 | Regression_eta: 0.0107 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9263 | Classification_binary: 3.2144 | Regression_cos_phi: 0.6110 | Regression_energy: 2.3034 | Regression_eta: 0.4493 | Regression_pt: 2.4975 | Regression_sin_phi: 0.7752 | LR: 9.56e-04 | DataLoad Time: 0.1419s | Model Forward Time: 1.1482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0034 | max_backward: 0.0342 | max_optimizer_step: 0.1092 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3397.6 | uss_MB: 2900.2 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2652.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:37:36,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 40.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:52,604] INFO : Step 7200/50000 rank0 | Train Loss: 10.6453 | Train Loss Unweighted: 3.7113 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.3190 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1688 | Regression_eta: 0.0098 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8429 | Classification_binary: 3.3190 | Regression_cos_phi: 0.4806 | Regression_energy: 2.2728 | Regression_eta: 0.4112 | Regression_pt: 2.4767 | Regression_sin_phi: 0.8420 | LR: 9.55e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2032 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3254.2 | uss_MB: 2757.7 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:37:52,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:08,717] INFO : Step 7300/50000 rank0 | Train Loss: 10.9155 | Train Loss Unweighted: 3.6839 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.2727 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1766 | Regression_eta: 0.0101 | Regression_pt: 0.1765 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9756 | Classification_binary: 3.2727 | Regression_cos_phi: 0.5119 | Regression_energy: 2.3773 | Regression_eta: 0.4252 | Regression_pt: 2.5813 | Regression_sin_phi: 0.7714 | LR: 9.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2006s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1932 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3254.2 | uss_MB: 2757.9 | child_rss_MB: 3219.0 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:38:08,942] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:24,928] INFO : Step 7400/50000 rank0 | Train Loss: 11.1385 | Train Loss Unweighted: 3.5861 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.1780 | Regression_cos_phi: 0.0118 | Regression_energy: 0.1735 | Regression_eta: 0.0117 | Regression_pt: 0.1708 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9001 | Classification_binary: 3.1780 | Regression_cos_phi: 0.6423 | Regression_energy: 2.3354 | Regression_eta: 0.4896 | Regression_pt: 2.4980 | Regression_sin_phi: 1.0950 | LR: 9.52e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1772s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0756 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.2956 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3254.2 | uss_MB: 2758.1 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:38:25,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:41,004] INFO : Step 7500/50000 rank0 | Train Loss: 11.0638 | Train Loss Unweighted: 3.6412 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.2261 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1806 | Regression_eta: 0.0118 | Regression_pt: 0.1741 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9014 | Classification_binary: 3.2261 | Regression_cos_phi: 0.6198 | Regression_energy: 2.4314 | Regression_eta: 0.4961 | Regression_pt: 2.5463 | Regression_sin_phi: 0.8426 | LR: 9.51e-04 | DataLoad Time: 0.0332s | Model Forward Time: 1.2194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.2047 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3254.2 | uss_MB: 2758.7 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:41,230] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:56,790] INFO : Step 7600/50000 rank0 | Train Loss: 10.2642 | Train Loss Unweighted: 3.5164 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.1436 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1590 | Regression_eta: 0.0113 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8040 | Classification_binary: 3.1436 | Regression_cos_phi: 0.5561 | Regression_energy: 2.1399 | Regression_eta: 0.4721 | Regression_pt: 2.3129 | Regression_sin_phi: 0.8356 | LR: 9.50e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0564 | max_forward: 0.0145 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1061 | max_tensorboard_logging: 0.3699 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3254.2 | uss_MB: 2758.8 | child_rss_MB: 3217.3 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:38:57,000] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:12,858] INFO : Step 7700/50000 rank0 | Train Loss: 11.1161 | Train Loss Unweighted: 3.7196 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.2971 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1827 | Regression_eta: 0.0112 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8649 | Classification_binary: 3.2971 | Regression_cos_phi: 0.5456 | Regression_energy: 2.4597 | Regression_eta: 0.4713 | Regression_pt: 2.6761 | Regression_sin_phi: 0.8015 | LR: 9.48e-04 | DataLoad Time: 0.0949s | Model Forward Time: 1.1715s | Phase Times: max_pre_step_barrier: 0.0647 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3254.2 | uss_MB: 2758.9 | child_rss_MB: 3214.9 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:39:13,070] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:28,887] INFO : Step 7800/50000 rank0 | Train Loss: 11.2819 | Train Loss Unweighted: 3.6456 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.2264 | Regression_cos_phi: 0.0135 | Regression_energy: 0.1764 | Regression_eta: 0.0145 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9625 | Classification_binary: 3.2264 | Regression_cos_phi: 0.7370 | Regression_energy: 2.3743 | Regression_eta: 0.6065 | Regression_pt: 2.5799 | Regression_sin_phi: 0.7953 | LR: 9.47e-04 | DataLoad Time: 0.0973s | Model Forward Time: 1.2379s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0452 | max_loss: 0.0033 | max_backward: 0.0350 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.3761 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3253.8 | uss_MB: 2758.6 | child_rss_MB: 3213.9 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:39:29,098] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:45,071] INFO : Step 7900/50000 rank0 | Train Loss: 11.0550 | Train Loss Unweighted: 3.5752 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0126 | Regression_energy: 0.1742 | Regression_eta: 0.0117 | Regression_pt: 0.1731 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8664 | Classification_binary: 3.1659 | Regression_cos_phi: 0.6852 | Regression_energy: 2.3445 | Regression_eta: 0.4918 | Regression_pt: 2.5315 | Regression_sin_phi: 0.9698 | LR: 9.46e-04 | DataLoad Time: 0.1254s | Model Forward Time: 1.2358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0294 | max_forward: 0.0151 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3253.8 | uss_MB: 2759.1 | child_rss_MB: 3216.2 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:39:45,283] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 11:40:01,184] INFO : Step 8000/50000 rank0 | Train Loss: 10.6105 | Train Loss Unweighted: 3.5674 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.1674 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1719 | Regression_eta: 0.0104 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9768 | Classification_binary: 3.1674 | Regression_cos_phi: 0.4904 | Regression_energy: 2.3146 | Regression_eta: 0.4339 | Regression_pt: 2.4958 | Regression_sin_phi: 0.7316 | LR: 9.44e-04 | DataLoad Time: 0.0853s | Model Forward Time: 1.2833s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0771 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3253.8 | uss_MB: 2759.2 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2772.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:40:01,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:01,463] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 11:40:01,544] INFO : Running validation on rank0 +rank0 [2026-10-28 11:40:01,544] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:40:01,544] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:40:02,618] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 11:40:02,761] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:40:02,892] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:40:02,966] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:40:03,034] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:40:03,102] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:40:03,171] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:40:03,240] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:40:03,350] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:40:03,481] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:40:03,611] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:40:03,682] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:40:03,751] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:40:03,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:40:03,888] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:40:03,957] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:40:04,065] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:40:04,071] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:40:04,074] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:40:04,075] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:40:04,078] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:40:04,080] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:40:04,081] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:40:04,084] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:40:04,086] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:40:04,087] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:40:04,090] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:40:04,091] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:40:04,094] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:40:04,096] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:40:04,097] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:40:04,099] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:40:09,887] INFO : VALIDATION | Step=8000/50000 | Train Loss=10.6105 | Valid Loss=10.7036 | Train Loss Unweighted=3.5674 | Valid Loss Unweighted=3.5624 | Stale=0 | ETA=144.9m +rank0 [2026-10-28 11:40:09,896] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:09,957] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:40:09,961] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:10,108] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:11,534] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:12,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:12,713] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:13,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:13,368] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:13,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:13,427] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:40:13,431] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:13,527] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:14,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:15,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:15,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:16,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:16,613] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:16,617] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:40:16,671] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:40:16,674] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:40:16,755] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:40:18,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:40:18,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:40:19,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:40:19,644] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:40:19,748] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:40:19,755] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:20,006] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:40:20,456] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:21,888] INFO : Plotted jets +rank0 [2026-10-28 11:40:22,970] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:23,001] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:23,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:40:24,042] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:25,429] INFO : Plotted jets +rank0 [2026-10-28 11:40:26,547] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:26,579] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 11:40:26,780] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 11:40:27,205] INFO : Plotted number of elements +rank0 [2026-10-28 11:40:28,601] INFO : Plotted jets +rank0 [2026-10-28 11:40:29,719] INFO : Plotted jet ratio +rank0 [2026-10-28 11:40:45,696] INFO : Step 8100/50000 rank0 | Train Loss: 10.6482 | Train Loss Unweighted: 3.5311 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.1403 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1644 | Regression_eta: 0.0114 | Regression_pt: 0.1668 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8861 | Classification_binary: 3.1403 | Regression_cos_phi: 0.5756 | Regression_energy: 2.2126 | Regression_eta: 0.4775 | Regression_pt: 2.4397 | Regression_sin_phi: 0.9165 | LR: 9.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2029s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0944 | max_optimizer_step: 0.1131 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3463.2 | uss_MB: 2970.4 | child_rss_MB: 3218.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:40:45,910] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:01,459] INFO : Step 8200/50000 rank0 | Train Loss: 10.4078 | Train Loss Unweighted: 3.6404 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.2632 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1597 | Regression_eta: 0.0109 | Regression_pt: 0.1617 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8726 | Classification_binary: 3.2632 | Regression_cos_phi: 0.4951 | Regression_energy: 2.1495 | Regression_eta: 0.4580 | Regression_pt: 2.3642 | Regression_sin_phi: 0.8052 | LR: 9.42e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0440 | max_forward: 0.0155 | max_loss: 0.0030 | max_backward: 0.0338 | max_optimizer_step: 0.1083 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3325.1 | uss_MB: 2832.1 | child_rss_MB: 3220.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:41:01,670] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:17,227] INFO : Step 8300/50000 rank0 | Train Loss: 10.2977 | Train Loss Unweighted: 3.5546 | Train Loss Components: Classification: 0.0244 | Classification_binary: 3.1660 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1671 | Regression_eta: 0.0102 | Regression_pt: 0.1694 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8283 | Classification_binary: 3.1660 | Regression_cos_phi: 0.5127 | Regression_energy: 2.2502 | Regression_eta: 0.4260 | Regression_pt: 2.4780 | Regression_sin_phi: 0.6365 | LR: 9.40e-04 | DataLoad Time: 0.0549s | Model Forward Time: 1.1751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0032 | max_backward: 0.0784 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.2913 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3326.1 | uss_MB: 2832.3 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:41:17,439] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:32,984] INFO : Step 8400/50000 rank0 | Train Loss: 10.7513 | Train Loss Unweighted: 3.5155 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1055 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1777 | Regression_eta: 0.0114 | Regression_pt: 0.1755 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8422 | Classification_binary: 3.1055 | Regression_cos_phi: 0.5666 | Regression_energy: 2.3924 | Regression_eta: 0.4779 | Regression_pt: 2.5673 | Regression_sin_phi: 0.7994 | LR: 9.39e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2580s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0648 | max_forward: 0.0155 | max_loss: 0.0030 | max_backward: 0.0340 | max_optimizer_step: 0.1048 | max_tensorboard_logging: 0.3726 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3326.1 | uss_MB: 2832.5 | child_rss_MB: 3214.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:33,199] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:48,696] INFO : Step 8500/50000 rank0 | Train Loss: 10.3485 | Train Loss Unweighted: 3.4752 | Train Loss Components: Classification: 0.0260 | Classification_binary: 3.0994 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1576 | Regression_eta: 0.0115 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8820 | Classification_binary: 3.0994 | Regression_cos_phi: 0.5769 | Regression_energy: 2.1220 | Regression_eta: 0.4806 | Regression_pt: 2.3288 | Regression_sin_phi: 0.8587 | LR: 9.37e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.2105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0325 | max_optimizer_step: 0.1990 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3326.1 | uss_MB: 2832.8 | child_rss_MB: 3217.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:41:48,923] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:04,190] INFO : Step 8600/50000 rank0 | Train Loss: 10.0229 | Train Loss Unweighted: 3.4751 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.1060 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1568 | Regression_eta: 0.0099 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8706 | Classification_binary: 3.1060 | Regression_cos_phi: 0.5192 | Regression_energy: 2.1110 | Regression_eta: 0.4147 | Regression_pt: 2.3191 | Regression_sin_phi: 0.6822 | LR: 9.36e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1790s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.1017 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3326.1 | uss_MB: 2832.9 | child_rss_MB: 3216.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:04,404] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:19,999] INFO : Step 8700/50000 rank0 | Train Loss: 9.9227 | Train Loss Unweighted: 3.3353 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.9779 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1502 | Regression_eta: 0.0111 | Regression_pt: 0.1499 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8722 | Classification_binary: 2.9779 | Regression_cos_phi: 0.5139 | Regression_energy: 2.0222 | Regression_eta: 0.4662 | Regression_pt: 2.1917 | Regression_sin_phi: 0.8787 | LR: 9.34e-04 | DataLoad Time: 0.1212s | Model Forward Time: 1.1749s | Phase Times: max_pre_step_barrier: 0.0551 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3327.1 | uss_MB: 2833.3 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:42:20,214] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:35,704] INFO : Step 8800/50000 rank0 | Train Loss: 11.5046 | Train Loss Unweighted: 3.6919 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.2655 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1785 | Regression_eta: 0.0149 | Regression_pt: 0.1801 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9619 | Classification_binary: 3.2655 | Regression_cos_phi: 0.7142 | Regression_energy: 2.4035 | Regression_eta: 0.6245 | Regression_pt: 2.6339 | Regression_sin_phi: 0.9011 | LR: 9.33e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2339s | Phase Times: max_pre_step_barrier: 0.0049 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3325.7 | uss_MB: 2832.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:42:35,935] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:42:51,401] INFO : Step 8900/50000 rank0 | Train Loss: 10.7739 | Train Loss Unweighted: 3.4304 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.0253 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1729 | Regression_eta: 0.0111 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9002 | Classification_binary: 3.0253 | Regression_cos_phi: 0.5458 | Regression_energy: 2.3275 | Regression_eta: 0.4669 | Regression_pt: 2.5159 | Regression_sin_phi: 0.9923 | LR: 9.31e-04 | DataLoad Time: 0.0926s | Model Forward Time: 1.3070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0786 | max_backward: 0.0319 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 3326.7 | uss_MB: 2832.0 | child_rss_MB: 3213.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:42:51,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 120.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:06,970] INFO : Step 9000/50000 rank0 | Train Loss: 9.8280 | Train Loss Unweighted: 3.4087 | Train Loss Components: Classification: 0.0270 | Classification_binary: 3.0559 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1480 | Regression_eta: 0.0109 | Regression_pt: 0.1483 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9151 | Classification_binary: 3.0559 | Regression_cos_phi: 0.5171 | Regression_energy: 1.9930 | Regression_eta: 0.4577 | Regression_pt: 2.1686 | Regression_sin_phi: 0.7206 | LR: 9.30e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0214 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3326.7 | uss_MB: 2833.4 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:43:07,184] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:07,258] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 11:43:07,340] INFO : Running validation on rank0 +rank0 [2026-10-28 11:43:07,340] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:43:07,341] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:43:08,434] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 11:43:08,577] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:43:08,708] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:43:08,781] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:43:08,849] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:43:08,916] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:43:08,985] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:43:09,055] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:43:09,162] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:43:09,285] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:43:09,405] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:43:09,477] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:43:09,547] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:43:09,614] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:43:09,682] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:43:09,750] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:43:09,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:43:09,871] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:43:09,873] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:43:09,874] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:43:09,877] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:43:09,879] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:43:09,881] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:43:09,883] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:43:09,886] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:43:09,887] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:43:09,890] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:43:09,892] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:43:09,893] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:43:09,896] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:43:09,896] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:43:09,899] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:43:15,951] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.8280 | Valid Loss=10.4704 | Train Loss Unweighted=3.4087 | Valid Loss Unweighted=3.5190 | Stale=0 | ETA=139.9m +rank0 [2026-10-28 11:43:15,962] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:16,026] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:43:16,029] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:16,179] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:17,610] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:18,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:18,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:19,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:19,392] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:19,395] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:19,451] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:43:19,454] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:19,534] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:20,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:21,450] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:21,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:22,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:22,710] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:22,714] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:43:22,767] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:43:22,770] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:43:22,827] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:43:24,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:43:24,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:43:25,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:43:25,900] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:43:26,003] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:43:26,011] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:26,262] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:43:26,718] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:28,156] INFO : Plotted jets +rank0 [2026-10-28 11:43:29,239] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:29,272] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:29,501] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:43:29,877] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:31,702] INFO : Plotted jets +rank0 [2026-10-28 11:43:32,811] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:32,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 11:43:33,044] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 11:43:33,459] INFO : Plotted number of elements +rank0 [2026-10-28 11:43:34,853] INFO : Plotted jets +rank0 [2026-10-28 11:43:35,941] INFO : Plotted jet ratio +rank0 [2026-10-28 11:43:51,678] INFO : Step 9100/50000 rank0 | Train Loss: 10.2210 | Train Loss Unweighted: 3.4537 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.0720 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1647 | Regression_eta: 0.0105 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7939 | Classification_binary: 3.0720 | Regression_cos_phi: 0.5389 | Regression_energy: 2.2175 | Regression_eta: 0.4383 | Regression_pt: 2.3909 | Regression_sin_phi: 0.7695 | LR: 9.28e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1523s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3081 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 3533.0 | uss_MB: 3040.3 | child_rss_MB: 3215.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:43:51,894] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:07,599] INFO : Step 9200/50000 rank0 | Train Loss: 10.2051 | Train Loss Unweighted: 3.4388 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.0578 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1628 | Regression_eta: 0.0101 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8480 | Classification_binary: 3.0578 | Regression_cos_phi: 0.5230 | Regression_energy: 2.1916 | Regression_eta: 0.4228 | Regression_pt: 2.3962 | Regression_sin_phi: 0.7658 | LR: 9.27e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2612s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0431 | max_loss: 0.0030 | max_backward: 0.0347 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3402.4 | uss_MB: 2908.4 | child_rss_MB: 3210.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:44:07,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:23,154] INFO : Step 9300/50000 rank0 | Train Loss: 9.7815 | Train Loss Unweighted: 3.4304 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.0844 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1460 | Regression_eta: 0.0100 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8392 | Classification_binary: 3.0844 | Regression_cos_phi: 0.5702 | Regression_energy: 1.9661 | Regression_eta: 0.4194 | Regression_pt: 2.1195 | Regression_sin_phi: 0.7828 | LR: 9.25e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.2869 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3402.4 | uss_MB: 2909.9 | child_rss_MB: 3214.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:44:23,370] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:38,575] INFO : Step 9400/50000 rank0 | Train Loss: 10.1318 | Train Loss Unweighted: 3.4839 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1170 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1558 | Regression_eta: 0.0112 | Regression_pt: 0.1540 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8917 | Classification_binary: 3.1170 | Regression_cos_phi: 0.5528 | Regression_energy: 2.0969 | Regression_eta: 0.4691 | Regression_pt: 2.2522 | Regression_sin_phi: 0.7521 | LR: 9.24e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1370s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3402.4 | uss_MB: 2910.1 | child_rss_MB: 3217.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:44:38,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:54,180] INFO : Step 9500/50000 rank0 | Train Loss: 10.3209 | Train Loss Unweighted: 3.4558 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.0716 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1654 | Regression_eta: 0.0113 | Regression_pt: 0.1625 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8669 | Classification_binary: 3.0716 | Regression_cos_phi: 0.5013 | Regression_energy: 2.2264 | Regression_eta: 0.4755 | Regression_pt: 2.3773 | Regression_sin_phi: 0.8019 | LR: 9.22e-04 | DataLoad Time: 0.0012s | Model Forward Time: 1.2126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0155 | max_loss: 0.0033 | max_backward: 0.1390 | max_optimizer_step: 0.1077 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3402.4 | uss_MB: 2910.2 | child_rss_MB: 3217.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:44:54,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:09,613] INFO : Step 9600/50000 rank0 | Train Loss: 10.6374 | Train Loss Unweighted: 3.4569 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.0632 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1671 | Regression_eta: 0.0120 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9034 | Classification_binary: 3.0632 | Regression_cos_phi: 0.6219 | Regression_energy: 2.2493 | Regression_eta: 0.5010 | Regression_pt: 2.4199 | Regression_sin_phi: 0.8787 | LR: 9.21e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1821s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0033 | max_backward: 0.0852 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3402.4 | uss_MB: 2910.3 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:09,825] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:25,324] INFO : Step 9700/50000 rank0 | Train Loss: 10.1423 | Train Loss Unweighted: 3.4407 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.0752 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1511 | Regression_eta: 0.0120 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8524 | Classification_binary: 3.0752 | Regression_cos_phi: 0.6542 | Regression_energy: 2.0336 | Regression_eta: 0.5017 | Regression_pt: 2.2804 | Regression_sin_phi: 0.7448 | LR: 9.19e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2420s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0403 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.0344 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3402.4 | uss_MB: 2910.4 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:25,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:40,776] INFO : Step 9800/50000 rank0 | Train Loss: 10.7870 | Train Loss Unweighted: 3.5506 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.1493 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1713 | Regression_eta: 0.0116 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8614 | Classification_binary: 3.1493 | Regression_cos_phi: 0.5457 | Regression_energy: 2.3058 | Regression_eta: 0.4876 | Regression_pt: 2.5032 | Regression_sin_phi: 0.9340 | LR: 9.17e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.1283s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0030 | max_backward: 0.0329 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.2904 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3400.1 | uss_MB: 2908.9 | child_rss_MB: 3214.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:40,993] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:56,065] INFO : Step 9900/50000 rank0 | Train Loss: 10.3658 | Train Loss Unweighted: 3.4258 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.0458 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1601 | Regression_eta: 0.0108 | Regression_pt: 0.1599 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9304 | Classification_binary: 3.0458 | Regression_cos_phi: 0.6235 | Regression_energy: 2.1556 | Regression_eta: 0.4533 | Regression_pt: 2.3386 | Regression_sin_phi: 0.8187 | LR: 9.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2939s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0797 | max_forward: 0.0154 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1085 | max_tensorboard_logging: 0.3813 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3402.1 | uss_MB: 2910.4 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:45:56,276] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:11,489] INFO : Step 10000/50000 rank0 | Train Loss: 12.1191 | Train Loss Unweighted: 3.6639 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.1969 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1976 | Regression_eta: 0.0164 | Regression_pt: 0.1997 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9570 | Classification_binary: 3.1969 | Regression_cos_phi: 0.6321 | Regression_energy: 2.6608 | Regression_eta: 0.6867 | Regression_pt: 2.9212 | Regression_sin_phi: 1.0643 | LR: 9.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0984 | max_forward: 0.0165 | max_loss: 0.0032 | max_backward: 0.0385 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3775 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3403.1 | uss_MB: 2912.9 | child_rss_MB: 3214.6 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 11:46:11,703] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:11,742] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 11:46:11,824] INFO : Running validation on rank0 +rank0 [2026-10-28 11:46:11,824] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:46:11,824] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:46:12,898] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 11:46:13,041] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:46:13,172] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:46:13,246] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:46:13,315] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:46:13,382] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:46:13,451] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:46:13,520] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:46:13,628] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:46:13,751] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:46:13,871] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:46:13,943] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:46:14,013] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:46:14,082] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:46:14,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:46:14,219] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:46:14,327] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:46:14,334] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:46:14,336] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:46:14,339] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:46:14,340] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:46:14,343] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:46:14,345] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:46:14,346] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:46:14,349] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:46:14,351] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:46:14,352] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:46:14,355] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:46:14,356] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:46:14,358] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:46:14,361] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:46:14,362] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:46:20,152] INFO : VALIDATION | Step=10000/50000 | Train Loss=12.1191 | Valid Loss=10.3040 | Train Loss Unweighted=3.6639 | Valid Loss Unweighted=3.4715 | Stale=0 | ETA=135.1m +rank0 [2026-10-28 11:46:20,164] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:20,227] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:46:20,231] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:20,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:21,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:22,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:22,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:23,529] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:23,620] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:23,623] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:23,680] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:46:23,683] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:23,768] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:25,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:25,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:26,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:26,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:27,055] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:27,058] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:46:27,111] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:46:27,115] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:46:27,178] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:46:28,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:46:29,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:46:29,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:46:30,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:46:30,167] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:46:30,174] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:30,423] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:46:30,875] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:32,318] INFO : Plotted jets +rank0 [2026-10-28 11:46:33,408] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:33,439] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:33,666] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:46:34,072] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:35,922] INFO : Plotted jets +rank0 [2026-10-28 11:46:37,038] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:37,201] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 11:46:37,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 11:46:37,825] INFO : Plotted number of elements +rank0 [2026-10-28 11:46:39,226] INFO : Plotted jets +rank0 [2026-10-28 11:46:40,320] INFO : Plotted jet ratio +rank0 [2026-10-28 11:46:55,629] INFO : Step 10100/50000 rank0 | Train Loss: 10.8258 | Train Loss Unweighted: 3.5885 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.1945 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1663 | Regression_eta: 0.0126 | Regression_pt: 0.1657 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8990 | Classification_binary: 3.1945 | Regression_cos_phi: 0.6024 | Regression_energy: 2.2392 | Regression_eta: 0.5295 | Regression_pt: 2.4227 | Regression_sin_phi: 0.9384 | LR: 9.12e-04 | DataLoad Time: 0.0229s | Model Forward Time: 1.2621s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.1341 | max_optimizer_step: 0.1143 | max_tensorboard_logging: 0.2898 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3609.3 | uss_MB: 3116.8 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:46:55,844] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:10,755] INFO : Step 10200/50000 rank0 | Train Loss: 10.2172 | Train Loss Unweighted: 3.4195 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.0523 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1542 | Regression_eta: 0.0120 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8723 | Classification_binary: 3.0523 | Regression_cos_phi: 0.6242 | Regression_energy: 2.0755 | Regression_eta: 0.5037 | Regression_pt: 2.2400 | Regression_sin_phi: 0.8493 | LR: 9.11e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0516 | max_forward: 0.0138 | max_loss: 0.0032 | max_backward: 0.0331 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3478.1 | uss_MB: 2985.7 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:47:10,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:26,041] INFO : Step 10300/50000 rank0 | Train Loss: 9.6742 | Train Loss Unweighted: 3.4153 | Train Loss Components: Classification: 0.0243 | Classification_binary: 3.0612 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1510 | Regression_eta: 0.0086 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 3.0612 | Regression_cos_phi: 0.4229 | Regression_energy: 2.0325 | Regression_eta: 0.3623 | Regression_pt: 2.2393 | Regression_sin_phi: 0.7298 | LR: 9.09e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3242s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1244 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1068 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3478.1 | uss_MB: 2987.1 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:26,255] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:41,367] INFO : Step 10400/50000 rank0 | Train Loss: 10.5939 | Train Loss Unweighted: 3.5083 | Train Loss Components: Classification: 0.0248 | Classification_binary: 3.1250 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1641 | Regression_eta: 0.0139 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8422 | Classification_binary: 3.1250 | Regression_cos_phi: 0.7059 | Regression_energy: 2.2085 | Regression_eta: 0.5829 | Regression_pt: 2.2959 | Regression_sin_phi: 0.8335 | LR: 9.07e-04 | DataLoad Time: 0.0092s | Model Forward Time: 1.3180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0138 | max_loss: 0.0788 | max_backward: 0.0332 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3478.1 | uss_MB: 2986.4 | child_rss_MB: 3216.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:41,581] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:56,232] INFO : Step 10500/50000 rank0 | Train Loss: 10.1197 | Train Loss Unweighted: 3.4256 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0461 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1629 | Regression_eta: 0.0097 | Regression_pt: 0.1624 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8907 | Classification_binary: 3.0461 | Regression_cos_phi: 0.5410 | Regression_energy: 2.1931 | Regression_eta: 0.4048 | Regression_pt: 2.3747 | Regression_sin_phi: 0.6693 | LR: 9.06e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1059 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3478.1 | uss_MB: 2987.8 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:47:56,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:11,295] INFO : Step 10600/50000 rank0 | Train Loss: 9.5424 | Train Loss Unweighted: 3.3392 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.9886 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1505 | Regression_eta: 0.0099 | Regression_pt: 0.1500 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7794 | Classification_binary: 2.9886 | Regression_cos_phi: 0.4746 | Regression_energy: 2.0263 | Regression_eta: 0.4162 | Regression_pt: 2.1944 | Regression_sin_phi: 0.6629 | LR: 9.04e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2139s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0032 | max_backward: 0.0683 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.2997 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3478.1 | uss_MB: 2987.9 | child_rss_MB: 3213.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:48:11,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:26,580] INFO : Step 10700/50000 rank0 | Train Loss: 9.3631 | Train Loss Unweighted: 3.2919 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9586 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1420 | Regression_eta: 0.0120 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7967 | Classification_binary: 2.9586 | Regression_cos_phi: 0.4644 | Regression_energy: 1.9119 | Regression_eta: 0.5020 | Regression_pt: 2.0242 | Regression_sin_phi: 0.7054 | LR: 9.02e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.2359s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.0345 | max_optimizer_step: 0.2115 | max_tensorboard_logging: 0.2897 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3478.1 | uss_MB: 2988.0 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:48:26,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:41,976] INFO : Step 10800/50000 rank0 | Train Loss: 9.8531 | Train Loss Unweighted: 3.3195 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9526 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1569 | Regression_eta: 0.0086 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8329 | Classification_binary: 2.9526 | Regression_cos_phi: 0.4783 | Regression_energy: 2.1123 | Regression_eta: 0.3586 | Regression_pt: 2.3082 | Regression_sin_phi: 0.8101 | LR: 9.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0029 | max_backward: 0.0331 | max_optimizer_step: 0.1939 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3478.1 | uss_MB: 2988.0 | child_rss_MB: 3215.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:42,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:57,412] INFO : Step 10900/50000 rank0 | Train Loss: 10.1457 | Train Loss Unweighted: 3.3871 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.0010 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1664 | Regression_eta: 0.0102 | Regression_pt: 0.1654 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.9144 | Classification_binary: 3.0010 | Regression_cos_phi: 0.4578 | Regression_energy: 2.2404 | Regression_eta: 0.4294 | Regression_pt: 2.4196 | Regression_sin_phi: 0.6830 | LR: 8.99e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2633s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0033 | max_backward: 0.0333 | max_optimizer_step: 0.2079 | max_tensorboard_logging: 0.3063 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3477.0 | uss_MB: 2987.4 | child_rss_MB: 3214.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:48:57,648] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:12,972] INFO : Step 11000/50000 rank0 | Train Loss: 10.1478 | Train Loss Unweighted: 3.3844 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.0058 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1620 | Regression_eta: 0.0106 | Regression_pt: 0.1611 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8561 | Classification_binary: 3.0058 | Regression_cos_phi: 0.5701 | Regression_energy: 2.1805 | Regression_eta: 0.4425 | Regression_pt: 2.3554 | Regression_sin_phi: 0.7373 | LR: 8.97e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2578s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0330 | max_optimizer_step: 0.2057 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0129 | Memory: rss_MB: 3477.0 | uss_MB: 2987.8 | child_rss_MB: 3214.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 11:49:13,206] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:49:13,223] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 11:49:13,303] INFO : Running validation on rank0 +rank0 [2026-10-28 11:49:13,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:49:13,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:49:14,496] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 11:49:14,640] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:49:14,771] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:49:14,844] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:49:14,912] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:49:14,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:49:15,048] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:49:15,116] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:49:15,228] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:49:15,358] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:49:15,488] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:49:15,558] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:49:15,626] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:49:15,693] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:49:15,762] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:49:15,830] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:49:15,938] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:49:15,945] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:49:15,947] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:49:15,950] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:49:15,951] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:49:15,954] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:49:15,957] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:49:15,960] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:49:15,961] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:49:15,964] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:49:15,967] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:49:15,970] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:49:15,971] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:49:15,973] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:49:15,974] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:49:15,977] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:49:21,902] INFO : VALIDATION | Step=11000/50000 | Train Loss=10.1478 | Valid Loss=10.1514 | Train Loss Unweighted=3.3844 | Valid Loss Unweighted=3.4104 | Stale=0 | ETA=130.5m +rank0 [2026-10-28 11:49:21,911] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:21,974] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:49:21,977] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:22,045] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:23,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:24,137] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:24,700] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:25,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:25,416] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:49:25,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:25,475] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:49:25,478] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:25,640] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:26,978] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:27,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:28,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:28,659] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:28,758] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:28,762] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:49:28,816] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:49:28,819] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:49:28,902] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:49:30,212] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:49:30,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:49:31,229] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:49:31,727] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:49:31,836] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:49:31,843] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:32,096] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:49:32,549] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:33,988] INFO : Plotted jets +rank0 [2026-10-28 11:49:35,071] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:35,103] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:35,328] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:35,705] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:37,549] INFO : Plotted jets +rank0 [2026-10-28 11:49:38,664] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:38,697] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:38,898] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:39,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:40,709] INFO : Plotted jets +rank0 [2026-10-28 11:49:41,805] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:57,070] INFO : Step 11100/50000 rank0 | Train Loss: 9.9509 | Train Loss Unweighted: 3.3484 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9855 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1554 | Regression_eta: 0.0107 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8041 | Classification_binary: 2.9855 | Regression_cos_phi: 0.6330 | Regression_energy: 2.0927 | Regression_eta: 0.4473 | Regression_pt: 2.2195 | Regression_sin_phi: 0.7687 | LR: 8.95e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2637s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0166 | max_loss: 0.0032 | max_backward: 0.1387 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3683.8 | uss_MB: 3196.4 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:49:57,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:12,186] INFO : Step 11200/50000 rank0 | Train Loss: 9.7615 | Train Loss Unweighted: 3.3100 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1483 | Regression_eta: 0.0117 | Regression_pt: 0.1472 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8991 | Classification_binary: 2.9575 | Regression_cos_phi: 0.4938 | Regression_energy: 1.9964 | Regression_eta: 0.4907 | Regression_pt: 2.1527 | Regression_sin_phi: 0.7712 | LR: 8.93e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0943 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1100 | max_tensorboard_logging: 0.3623 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3683.8 | uss_MB: 3196.5 | child_rss_MB: 3211.0 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:50:12,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:26,912] INFO : Step 11300/50000 rank0 | Train Loss: 9.8666 | Train Loss Unweighted: 3.3011 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9421 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1533 | Regression_eta: 0.0129 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7992 | Classification_binary: 2.9421 | Regression_cos_phi: 0.5864 | Regression_energy: 2.0637 | Regression_eta: 0.5403 | Regression_pt: 2.1794 | Regression_sin_phi: 0.7556 | LR: 8.91e-04 | DataLoad Time: 0.1135s | Model Forward Time: 1.1713s | Phase Times: max_pre_step_barrier: 0.0064 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1087 | max_tensorboard_logging: 0.3009 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 3548.8 | uss_MB: 3063.0 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:50:27,124] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:41,706] INFO : Step 11400/50000 rank0 | Train Loss: 9.7117 | Train Loss Unweighted: 3.3258 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9925 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1367 | Regression_eta: 0.0139 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8040 | Classification_binary: 2.9925 | Regression_cos_phi: 0.6689 | Regression_energy: 1.8404 | Regression_eta: 0.5832 | Regression_pt: 1.9909 | Regression_sin_phi: 0.8317 | LR: 8.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1181s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1091 | max_tensorboard_logging: 0.2847 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3549.8 | uss_MB: 3063.1 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:50:41,924] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:56,686] INFO : Step 11500/50000 rank0 | Train Loss: 10.2846 | Train Loss Unweighted: 3.3150 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.9364 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1591 | Regression_eta: 0.0161 | Regression_pt: 0.1579 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8368 | Classification_binary: 2.9364 | Regression_cos_phi: 0.5886 | Regression_energy: 2.1423 | Regression_eta: 0.6741 | Regression_pt: 2.3096 | Regression_sin_phi: 0.7969 | LR: 8.88e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0616 | max_optimizer_step: 0.1127 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3550.8 | uss_MB: 3063.3 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:50:56,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:11,731] INFO : Step 11600/50000 rank0 | Train Loss: 10.0478 | Train Loss Unweighted: 3.2852 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9071 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1616 | Regression_eta: 0.0112 | Regression_pt: 0.1609 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8658 | Classification_binary: 2.9071 | Regression_cos_phi: 0.5044 | Regression_energy: 2.1749 | Regression_eta: 0.4687 | Regression_pt: 2.3527 | Regression_sin_phi: 0.7742 | LR: 8.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2831s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0530 | max_loss: 0.0031 | max_backward: 0.0325 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3890 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 3551.8 | uss_MB: 3063.4 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:51:11,948] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:26,575] INFO : Step 11700/50000 rank0 | Train Loss: 10.6136 | Train Loss Unweighted: 3.4505 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0729 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1599 | Regression_eta: 0.0145 | Regression_pt: 0.1521 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8882 | Classification_binary: 3.0729 | Regression_cos_phi: 0.6670 | Regression_energy: 2.1524 | Regression_eta: 0.6077 | Regression_pt: 2.2242 | Regression_sin_phi: 1.0011 | LR: 8.84e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.1359s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3551.8 | uss_MB: 3063.6 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:51:26,791] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:41,174] INFO : Step 11800/50000 rank0 | Train Loss: 11.0674 | Train Loss Unweighted: 3.4144 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9898 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1837 | Regression_eta: 0.0145 | Regression_pt: 0.1794 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8057 | Classification_binary: 2.9898 | Regression_cos_phi: 0.6092 | Regression_energy: 2.4730 | Regression_eta: 0.6092 | Regression_pt: 2.6231 | Regression_sin_phi: 0.9574 | LR: 8.82e-04 | DataLoad Time: 0.0644s | Model Forward Time: 1.2657s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0545 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3491 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 3551.8 | uss_MB: 3063.6 | child_rss_MB: 3209.5 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:51:41,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:56,465] INFO : Step 11900/50000 rank0 | Train Loss: 9.2926 | Train Loss Unweighted: 3.2014 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.8618 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1440 | Regression_eta: 0.0075 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8465 | Classification_binary: 2.8618 | Regression_cos_phi: 0.5379 | Regression_energy: 1.9386 | Regression_eta: 0.3161 | Regression_pt: 2.1163 | Regression_sin_phi: 0.6754 | LR: 8.80e-04 | DataLoad Time: 0.0016s | Model Forward Time: 1.2869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1003 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1070 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3550.9 | uss_MB: 3063.1 | child_rss_MB: 3209.2 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:51:56,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:11,214] INFO : Step 12000/50000 rank0 | Train Loss: 9.3280 | Train Loss Unweighted: 3.2430 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.8970 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1491 | Regression_eta: 0.0097 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8079 | Classification_binary: 2.8970 | Regression_cos_phi: 0.4599 | Regression_energy: 2.0078 | Regression_eta: 0.4056 | Regression_pt: 2.1564 | Regression_sin_phi: 0.5933 | LR: 8.78e-04 | DataLoad Time: 0.1076s | Model Forward Time: 1.3308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0977 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3884 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3550.9 | uss_MB: 3063.2 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2736.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:52:11,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:11,541] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 11:52:11,624] INFO : Running validation on rank0 +rank0 [2026-10-28 11:52:11,624] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:52:11,625] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:52:12,716] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 11:52:12,859] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:52:12,991] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:52:13,067] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:52:13,138] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:52:13,206] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:52:13,274] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:52:13,344] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:52:13,455] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:52:13,587] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:52:13,718] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:52:13,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:52:13,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:52:13,928] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:52:13,998] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:52:14,068] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:52:14,183] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:52:14,190] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:52:14,191] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:52:14,194] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:52:14,197] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:52:14,199] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:52:14,201] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:52:14,203] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:52:14,204] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:52:14,207] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:52:14,210] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:52:14,212] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:52:14,215] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:52:14,217] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:52:14,219] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:52:14,220] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:52:20,305] INFO : VALIDATION | Step=12000/50000 | Train Loss=9.3280 | Valid Loss=9.9684 | Train Loss Unweighted=3.2430 | Valid Loss Unweighted=3.3511 | Stale=0 | ETA=126.0m +rank0 [2026-10-28 11:52:20,320] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:20,382] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:52:20,385] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:20,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:22,022] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:22,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:23,221] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:23,883] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:23,981] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:52:23,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:24,039] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:52:24,043] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:24,128] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:25,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:26,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:26,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:27,251] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:27,345] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:27,348] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:52:27,402] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:52:27,405] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:52:27,492] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:52:28,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:52:29,335] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:52:29,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:52:30,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:52:30,458] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:52:30,467] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:30,717] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:52:31,169] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:32,609] INFO : Plotted jets +rank0 [2026-10-28 11:52:33,692] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:33,725] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:33,949] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:52:34,327] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:36,165] INFO : Plotted jets +rank0 [2026-10-28 11:52:37,294] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:37,327] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 11:52:37,529] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 11:52:37,950] INFO : Plotted number of elements +rank0 [2026-10-28 11:52:39,351] INFO : Plotted jets +rank0 [2026-10-28 11:52:40,453] INFO : Plotted jet ratio +rank0 [2026-10-28 11:52:55,408] INFO : Step 12100/50000 rank0 | Train Loss: 10.0726 | Train Loss Unweighted: 3.3186 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9450 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1590 | Regression_eta: 0.0131 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.9450 | Regression_cos_phi: 0.5372 | Regression_energy: 2.1408 | Regression_eta: 0.5487 | Regression_pt: 2.2918 | Regression_sin_phi: 0.7418 | LR: 8.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2512s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0343 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1134 | max_tensorboard_logging: 0.3459 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3763.5 | uss_MB: 3279.0 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:52:55,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:10,087] INFO : Step 12200/50000 rank0 | Train Loss: 9.7603 | Train Loss Unweighted: 3.3390 | Train Loss Components: Classification: 0.0260 | Classification_binary: 2.9841 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1500 | Regression_eta: 0.0110 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8837 | Classification_binary: 2.9841 | Regression_cos_phi: 0.4914 | Regression_energy: 2.0199 | Regression_eta: 0.4593 | Regression_pt: 2.1876 | Regression_sin_phi: 0.7342 | LR: 8.74e-04 | DataLoad Time: 0.1000s | Model Forward Time: 1.2326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0227 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3406 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3763.5 | uss_MB: 3279.1 | child_rss_MB: 3211.8 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:53:10,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:24,733] INFO : Step 12300/50000 rank0 | Train Loss: 9.8808 | Train Loss Unweighted: 3.3626 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9892 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1614 | Regression_eta: 0.0096 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7926 | Classification_binary: 2.9892 | Regression_cos_phi: 0.4996 | Regression_energy: 2.1726 | Regression_eta: 0.4029 | Regression_pt: 2.3618 | Regression_sin_phi: 0.6620 | LR: 8.72e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2966s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0877 | max_loss: 0.0032 | max_backward: 0.0332 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.3757 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3624.4 | uss_MB: 3140.0 | child_rss_MB: 3209.9 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:24,947] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:39,186] INFO : Step 12400/50000 rank0 | Train Loss: 9.9525 | Train Loss Unweighted: 3.3426 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9756 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1557 | Regression_eta: 0.0150 | Regression_pt: 0.1559 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7377 | Classification_binary: 2.9756 | Regression_cos_phi: 0.5284 | Regression_energy: 2.0967 | Regression_eta: 0.6285 | Regression_pt: 2.2796 | Regression_sin_phi: 0.7060 | LR: 8.70e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2258s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0032 | max_backward: 0.1389 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3625.4 | uss_MB: 3140.3 | child_rss_MB: 3208.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:39,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:53,616] INFO : Step 12500/50000 rank0 | Train Loss: 10.6162 | Train Loss Unweighted: 3.3865 | Train Loss Components: Classification: 0.0253 | Classification_binary: 2.9762 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1777 | Regression_eta: 0.0114 | Regression_pt: 0.1756 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8601 | Classification_binary: 2.9762 | Regression_cos_phi: 0.5689 | Regression_energy: 2.3926 | Regression_eta: 0.4779 | Regression_pt: 2.5684 | Regression_sin_phi: 0.7720 | LR: 8.68e-04 | DataLoad Time: 0.0939s | Model Forward Time: 1.1895s | Phase Times: max_pre_step_barrier: 0.0653 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1102 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3625.4 | uss_MB: 3140.6 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:53:53,830] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:08,267] INFO : Step 12600/50000 rank0 | Train Loss: 10.0286 | Train Loss Unweighted: 3.2811 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.9085 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1597 | Regression_eta: 0.0139 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7704 | Classification_binary: 2.9085 | Regression_cos_phi: 0.4935 | Regression_energy: 2.1501 | Regression_eta: 0.5809 | Regression_pt: 2.2935 | Regression_sin_phi: 0.8317 | LR: 8.66e-04 | DataLoad Time: 0.1165s | Model Forward Time: 1.3459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1178 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3625.4 | uss_MB: 3140.8 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:08,478] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 121.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:22,649] INFO : Step 12700/50000 rank0 | Train Loss: 9.7353 | Train Loss Unweighted: 3.2943 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.9402 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1513 | Regression_eta: 0.0142 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7772 | Classification_binary: 2.9402 | Regression_cos_phi: 0.4766 | Regression_energy: 2.0374 | Regression_eta: 0.5958 | Regression_pt: 2.1567 | Regression_sin_phi: 0.7515 | LR: 8.64e-04 | DataLoad Time: 0.0567s | Model Forward Time: 1.1275s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0542 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2685 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3625.4 | uss_MB: 3140.9 | child_rss_MB: 3211.1 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:54:22,863] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:37,358] INFO : Step 12800/50000 rank0 | Train Loss: 9.5525 | Train Loss Unweighted: 3.3974 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.0452 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1521 | Regression_eta: 0.0097 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 3.0452 | Regression_cos_phi: 0.4103 | Regression_energy: 2.0476 | Regression_eta: 0.4083 | Regression_pt: 2.1962 | Regression_sin_phi: 0.5863 | LR: 8.62e-04 | DataLoad Time: 0.0946s | Model Forward Time: 1.1456s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0557 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3625.4 | uss_MB: 3140.9 | child_rss_MB: 3208.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:54:37,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:51,753] INFO : Step 12900/50000 rank0 | Train Loss: 9.7605 | Train Loss Unweighted: 3.3315 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.9757 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1511 | Regression_eta: 0.0102 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8169 | Classification_binary: 2.9757 | Regression_cos_phi: 0.5302 | Regression_energy: 2.0340 | Regression_eta: 0.4256 | Regression_pt: 2.2079 | Regression_sin_phi: 0.7702 | LR: 8.60e-04 | DataLoad Time: 0.1048s | Model Forward Time: 1.1621s | Phase Times: max_pre_step_barrier: 0.0203 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3010 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3623.9 | uss_MB: 3140.3 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:54:51,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:06,361] INFO : Step 13000/50000 rank0 | Train Loss: 10.4666 | Train Loss Unweighted: 3.4348 | Train Loss Components: Classification: 0.0247 | Classification_binary: 3.0425 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1681 | Regression_eta: 0.0111 | Regression_pt: 0.1673 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8399 | Classification_binary: 3.0425 | Regression_cos_phi: 0.5639 | Regression_energy: 2.2626 | Regression_eta: 0.4651 | Regression_pt: 2.4472 | Regression_sin_phi: 0.8455 | LR: 8.58e-04 | DataLoad Time: 0.0927s | Model Forward Time: 1.2725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0729 | max_loss: 0.0031 | max_backward: 0.0333 | max_optimizer_step: 0.1106 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3623.9 | uss_MB: 3140.5 | child_rss_MB: 3207.6 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:55:06,573] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:06,591] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 11:55:06,778] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:06,779] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:06,779] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:07,855] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 11:55:07,999] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:08,131] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:08,205] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:08,276] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:08,344] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:08,414] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:08,483] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:08,594] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:08,716] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:08,839] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:08,912] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:08,982] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:09,050] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:09,119] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:09,190] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:09,299] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:09,306] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:09,309] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:09,312] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:09,312] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:09,315] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:09,318] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:09,319] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:09,322] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:09,323] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:09,325] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:09,328] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:09,331] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:09,332] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:09,335] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:09,338] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:15,353] INFO : VALIDATION | Step=13000/50000 | Train Loss=10.4666 | Valid Loss=9.8645 | Train Loss Unweighted=3.4348 | Valid Loss Unweighted=3.2896 | Stale=0 | ETA=121.5m +rank0 [2026-10-28 11:55:15,363] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:15,424] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:15,428] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:15,579] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:17,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:17,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:18,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:19,017] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:19,117] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:55:19,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:19,176] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:55:19,179] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:19,257] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:20,686] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:21,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:21,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:22,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:22,482] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:22,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:22,538] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:55:22,540] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:55:22,632] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:55:24,050] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:55:24,566] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:55:25,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:55:25,584] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:55:25,689] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:55:25,696] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:25,946] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:26,397] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:27,834] INFO : Plotted jets +rank0 [2026-10-28 11:55:28,918] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:28,949] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:29,171] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:55:29,549] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:31,353] INFO : Plotted jets +rank0 [2026-10-28 11:55:32,467] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:32,499] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 11:55:32,699] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 11:55:33,114] INFO : Plotted number of elements +rank0 [2026-10-28 11:55:34,505] INFO : Plotted jets +rank0 [2026-10-28 11:55:35,593] INFO : Plotted jet ratio +rank0 [2026-10-28 11:55:49,999] INFO : Step 13100/50000 rank0 | Train Loss: 9.9021 | Train Loss Unweighted: 3.3099 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9571 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1481 | Regression_eta: 0.0102 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7999 | Classification_binary: 2.9571 | Regression_cos_phi: 0.5824 | Regression_energy: 1.9936 | Regression_eta: 0.4293 | Regression_pt: 2.1616 | Regression_sin_phi: 0.9780 | LR: 8.56e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1192s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0133 | max_loss: 0.0032 | max_backward: 0.0325 | max_optimizer_step: 0.1094 | max_tensorboard_logging: 0.2712 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3814.4 | uss_MB: 3332.7 | child_rss_MB: 3205.8 | allocated_MB: 207.0 | reserved_MB: 2630.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 11:55:50,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:04,489] INFO : Step 13200/50000 rank0 | Train Loss: 9.4704 | Train Loss Unweighted: 3.2823 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9415 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1459 | Regression_eta: 0.0109 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7440 | Classification_binary: 2.9415 | Regression_cos_phi: 0.4974 | Regression_energy: 1.9648 | Regression_eta: 0.4579 | Regression_pt: 2.0933 | Regression_sin_phi: 0.7714 | LR: 8.54e-04 | DataLoad Time: 0.0987s | Model Forward Time: 1.3084s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0767 | max_backward: 0.0337 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3730 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3814.4 | uss_MB: 3331.5 | child_rss_MB: 3208.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:56:04,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:18,717] INFO : Step 13300/50000 rank0 | Train Loss: 10.2088 | Train Loss Unweighted: 3.3526 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.9630 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1679 | Regression_eta: 0.0100 | Regression_pt: 0.1677 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8618 | Classification_binary: 2.9630 | Regression_cos_phi: 0.5021 | Regression_energy: 2.2602 | Regression_eta: 0.4202 | Regression_pt: 2.4519 | Regression_sin_phi: 0.7497 | LR: 8.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0615 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3694.6 | uss_MB: 3211.4 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:56:18,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:33,044] INFO : Step 13400/50000 rank0 | Train Loss: 9.8936 | Train Loss Unweighted: 3.3379 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.9810 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1510 | Regression_eta: 0.0114 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7942 | Classification_binary: 2.9810 | Regression_cos_phi: 0.6452 | Regression_energy: 2.0328 | Regression_eta: 0.4778 | Regression_pt: 2.1852 | Regression_sin_phi: 0.7775 | LR: 8.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2317s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0032 | max_backward: 0.1153 | max_optimizer_step: 0.1111 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3694.6 | uss_MB: 3212.4 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:56:33,258] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:56:47,006] INFO : Step 13500/50000 rank0 | Train Loss: 10.1506 | Train Loss Unweighted: 3.3300 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.9547 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1593 | Regression_eta: 0.0142 | Regression_pt: 0.1581 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.9547 | Regression_cos_phi: 0.5353 | Regression_energy: 2.1441 | Regression_eta: 0.5945 | Regression_pt: 2.3128 | Regression_sin_phi: 0.8032 | LR: 8.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.0164 | max_loss: 0.0032 | max_backward: 0.0721 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2746 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3695.6 | uss_MB: 3215.3 | child_rss_MB: 3210.5 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:56:47,220] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:01,604] INFO : Step 13600/50000 rank0 | Train Loss: 10.3087 | Train Loss Unweighted: 3.3338 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.9396 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1709 | Regression_eta: 0.0117 | Regression_pt: 0.1685 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7776 | Classification_binary: 2.9396 | Regression_cos_phi: 0.5880 | Regression_energy: 2.3003 | Regression_eta: 0.4887 | Regression_pt: 2.4650 | Regression_sin_phi: 0.7496 | LR: 8.45e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2389s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0033 | max_backward: 0.0340 | max_optimizer_step: 0.2069 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3695.6 | uss_MB: 3213.4 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:01,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:15,689] INFO : Step 13700/50000 rank0 | Train Loss: 10.0445 | Train Loss Unweighted: 3.3482 | Train Loss Components: Classification: 0.0262 | Classification_binary: 2.9761 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1580 | Regression_eta: 0.0116 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.9761 | Regression_cos_phi: 0.5853 | Regression_energy: 2.1275 | Regression_eta: 0.4880 | Regression_pt: 2.2931 | Regression_sin_phi: 0.6851 | LR: 8.43e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2143s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0179 | max_loss: 0.0033 | max_backward: 0.1040 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3695.6 | uss_MB: 3215.7 | child_rss_MB: 3211.3 | allocated_MB: 210.7 | reserved_MB: 2750.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 11:57:15,904] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:29,967] INFO : Step 13800/50000 rank0 | Train Loss: 9.8036 | Train Loss Unweighted: 3.2180 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.8653 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1484 | Regression_eta: 0.0113 | Regression_pt: 0.1461 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8381 | Classification_binary: 2.8653 | Regression_cos_phi: 0.5862 | Regression_energy: 1.9976 | Regression_eta: 0.4718 | Regression_pt: 2.1371 | Regression_sin_phi: 0.9077 | LR: 8.41e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3250s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1174 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3787 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3695.6 | uss_MB: 3213.7 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:30,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:44,292] INFO : Step 13900/50000 rank0 | Train Loss: 9.5682 | Train Loss Unweighted: 3.3080 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.9676 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1446 | Regression_eta: 0.0114 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8158 | Classification_binary: 2.9676 | Regression_cos_phi: 0.4844 | Regression_energy: 1.9467 | Regression_eta: 0.4760 | Regression_pt: 2.0648 | Regression_sin_phi: 0.8128 | LR: 8.39e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.2133 | max_tensorboard_logging: 0.2953 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3694.1 | uss_MB: 3211.7 | child_rss_MB: 3213.1 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:57:44,522] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 122.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:58,373] INFO : Step 14000/50000 rank0 | Train Loss: 9.9458 | Train Loss Unweighted: 3.2759 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9074 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1601 | Regression_eta: 0.0135 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.9074 | Regression_cos_phi: 0.5622 | Regression_energy: 2.1548 | Regression_eta: 0.5651 | Regression_pt: 2.2421 | Regression_sin_phi: 0.7881 | LR: 8.37e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1722s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0678 | max_optimizer_step: 0.1125 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3695.1 | uss_MB: 3212.7 | child_rss_MB: 3212.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:57:58,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:58,607] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 11:57:58,687] INFO : Running validation on rank0 +rank0 [2026-10-28 11:57:58,688] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:57:58,688] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:57:59,871] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 11:58:00,016] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:58:00,148] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:58:00,222] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:58:00,290] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:58:00,358] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:58:00,428] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:58:00,499] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:58:00,607] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:58:00,730] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:58:00,853] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:58:00,925] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:58:00,993] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:58:01,063] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:58:01,134] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:58:01,204] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:58:01,311] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:58:01,317] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:58:01,321] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:58:01,324] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:58:01,325] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:58:01,328] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:58:01,331] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:58:01,332] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:58:01,335] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:58:01,337] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:58:01,340] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:58:01,341] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:58:01,343] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:58:01,345] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:58:01,347] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:58:01,350] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:58:07,199] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.9458 | Valid Loss=9.9386 | Train Loss Unweighted=3.2759 | Valid Loss Unweighted=3.2938 | Stale=1 | ETA=117.2m +rank0 [2026-10-28 11:58:07,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:07,270] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:58:07,273] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:07,337] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:08,819] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:09,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:10,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:10,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:10,749] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:58:10,752] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:10,809] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:58:10,811] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:10,969] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:12,360] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:12,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:13,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:14,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:14,230] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:14,233] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:58:14,287] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:58:14,303] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:58:14,459] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:58:15,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:58:16,305] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:58:16,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:58:17,439] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:58:17,529] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 11:58:17,537] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:17,862] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:58:18,316] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:19,760] INFO : Plotted jets +rank0 [2026-10-28 11:58:20,853] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:20,888] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:21,114] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:58:21,494] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:23,307] INFO : Plotted jets +rank0 [2026-10-28 11:58:24,431] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:24,465] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 11:58:24,663] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 11:58:25,083] INFO : Plotted number of elements +rank0 [2026-10-28 11:58:26,490] INFO : Plotted jets +rank0 [2026-10-28 11:58:27,604] INFO : Plotted jet ratio +rank0 [2026-10-28 11:58:41,581] INFO : Step 14100/50000 rank0 | Train Loss: 10.0267 | Train Loss Unweighted: 3.2078 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8400 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1546 | Regression_eta: 0.0118 | Regression_pt: 0.1543 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.8400 | Regression_cos_phi: 0.5396 | Regression_energy: 2.0811 | Regression_eta: 0.4950 | Regression_pt: 2.2569 | Regression_sin_phi: 0.9638 | LR: 8.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2002s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0833 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.2730 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3903.4 | uss_MB: 3420.7 | child_rss_MB: 3211.4 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:58:41,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:55,861] INFO : Step 14200/50000 rank0 | Train Loss: 10.3101 | Train Loss Unweighted: 3.3419 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.9599 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1653 | Regression_eta: 0.0141 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7757 | Classification_binary: 2.9599 | Regression_cos_phi: 0.5479 | Regression_energy: 2.2250 | Regression_eta: 0.5898 | Regression_pt: 2.3156 | Regression_sin_phi: 0.8962 | LR: 8.32e-04 | DataLoad Time: 0.1271s | Model Forward Time: 1.2848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0740 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.3673 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3903.4 | uss_MB: 3420.8 | child_rss_MB: 3212.7 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:58:56,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:09,912] INFO : Step 14300/50000 rank0 | Train Loss: 10.8830 | Train Loss Unweighted: 3.3773 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.9602 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1742 | Regression_eta: 0.0168 | Regression_pt: 0.1803 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8642 | Classification_binary: 2.9602 | Regression_cos_phi: 0.5206 | Regression_energy: 2.3449 | Regression_eta: 0.7049 | Regression_pt: 2.6367 | Regression_sin_phi: 0.8515 | LR: 8.30e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.3341s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1051 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1091 | max_tensorboard_logging: 0.3620 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3769.1 | uss_MB: 3286.2 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:59:10,127] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:24,198] INFO : Step 14400/50000 rank0 | Train Loss: 9.9240 | Train Loss Unweighted: 3.2902 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.9117 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1644 | Regression_eta: 0.0113 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8286 | Classification_binary: 2.9117 | Regression_cos_phi: 0.4625 | Regression_energy: 2.2138 | Regression_eta: 0.4756 | Regression_pt: 2.3593 | Regression_sin_phi: 0.6723 | LR: 8.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.1406 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2752 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3769.1 | uss_MB: 3287.3 | child_rss_MB: 3213.0 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 11:59:24,410] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:38,495] INFO : Step 14500/50000 rank0 | Train Loss: 9.5049 | Train Loss Unweighted: 3.1723 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.8152 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1517 | Regression_eta: 0.0093 | Regression_pt: 0.1541 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8140 | Classification_binary: 2.8152 | Regression_cos_phi: 0.5138 | Regression_energy: 2.0427 | Regression_eta: 0.3890 | Regression_pt: 2.2532 | Regression_sin_phi: 0.6769 | LR: 8.26e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0738 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1090 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3769.1 | uss_MB: 3287.4 | child_rss_MB: 3214.0 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 11:59:38,707] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:52,395] INFO : Step 14600/50000 rank0 | Train Loss: 9.9713 | Train Loss Unweighted: 3.2409 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8667 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1596 | Regression_eta: 0.0110 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8127 | Classification_binary: 2.8667 | Regression_cos_phi: 0.5104 | Regression_energy: 2.1480 | Regression_eta: 0.4624 | Regression_pt: 2.3361 | Regression_sin_phi: 0.8350 | LR: 8.24e-04 | DataLoad Time: 0.0851s | Model Forward Time: 1.3219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0826 | max_backward: 0.0328 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3769.1 | uss_MB: 3286.8 | child_rss_MB: 3212.6 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 11:59:52,607] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:06,144] INFO : Step 14700/50000 rank0 | Train Loss: 9.7108 | Train Loss Unweighted: 3.2602 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8926 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1589 | Regression_eta: 0.0123 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7975 | Classification_binary: 2.8926 | Regression_cos_phi: 0.4664 | Regression_energy: 2.1397 | Regression_eta: 0.5149 | Regression_pt: 2.2919 | Regression_sin_phi: 0.6078 | LR: 8.21e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.1850s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0031 | max_backward: 0.0337 | max_optimizer_step: 0.1752 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3769.1 | uss_MB: 3287.9 | child_rss_MB: 3213.3 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:06,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:19,933] INFO : Step 14800/50000 rank0 | Train Loss: 10.4445 | Train Loss Unweighted: 3.3912 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.0009 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1689 | Regression_eta: 0.0132 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8549 | Classification_binary: 3.0009 | Regression_cos_phi: 0.5854 | Regression_energy: 2.2740 | Regression_eta: 0.5543 | Regression_pt: 2.3690 | Regression_sin_phi: 0.8059 | LR: 8.19e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1389s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3769.1 | uss_MB: 3288.1 | child_rss_MB: 3212.4 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:00:20,147] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:34,128] INFO : Step 14900/50000 rank0 | Train Loss: 10.1577 | Train Loss Unweighted: 3.3607 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.9874 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1596 | Regression_eta: 0.0115 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8101 | Classification_binary: 2.9874 | Regression_cos_phi: 0.5326 | Regression_energy: 2.1488 | Regression_eta: 0.4829 | Regression_pt: 2.2996 | Regression_sin_phi: 0.8963 | LR: 8.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.2084 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3768.0 | uss_MB: 3286.5 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:00:34,358] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:47,891] INFO : Step 15000/50000 rank0 | Train Loss: 8.5458 | Train Loss Unweighted: 3.0886 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.7865 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1271 | Regression_eta: 0.0103 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8249 | Classification_binary: 2.7865 | Regression_cos_phi: 0.3969 | Regression_energy: 1.7109 | Regression_eta: 0.4336 | Regression_pt: 1.8447 | Regression_sin_phi: 0.5482 | LR: 8.15e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.1365s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1102 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3769.0 | uss_MB: 3287.6 | child_rss_MB: 3212.8 | allocated_MB: 207.0 | reserved_MB: 2740.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:00:48,104] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:48,151] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 12:00:48,232] INFO : Running validation on rank0 +rank0 [2026-10-28 12:00:48,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:00:48,233] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:00:49,325] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 12:00:49,470] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:00:49,602] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:00:49,677] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:00:49,746] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:00:49,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:00:49,884] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:00:49,955] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:00:50,065] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:00:50,199] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:00:50,331] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:00:50,403] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:00:50,473] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:00:50,542] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:00:50,612] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:00:50,682] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:00:50,793] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:00:50,800] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:00:50,803] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:00:50,806] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:00:50,807] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:00:50,810] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:00:50,812] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:00:50,815] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:00:50,816] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:00:50,818] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:00:50,819] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:00:50,822] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:00:50,932] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:00:50,935] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:00:50,938] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:00:50,941] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:00:56,781] INFO : VALIDATION | Step=15000/50000 | Train Loss=8.5458 | Valid Loss=9.7435 | Train Loss Unweighted=3.0886 | Valid Loss Unweighted=3.2473 | Stale=0 | ETA=112.9m +rank0 [2026-10-28 12:00:56,791] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:00:56,852] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:00:56,855] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:00:57,013] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:00:58,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:00:59,103] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:00:59,777] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:00,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:00,440] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:01:00,443] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:00,499] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:01:00,503] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:00,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:01,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:02,483] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:03,018] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:03,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:03,653] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:03,655] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:01:03,709] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:01:03,712] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:01:03,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:01:05,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:01:05,788] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:01:06,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:01:06,806] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:01:06,937] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:01:06,944] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:01:07,193] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:01:07,646] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:09,087] INFO : Plotted jets +rank0 [2026-10-28 12:01:10,170] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:10,202] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:10,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:01:10,809] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:12,647] INFO : Plotted jets +rank0 [2026-10-28 12:01:13,763] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:13,795] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 12:01:13,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 12:01:14,415] INFO : Plotted number of elements +rank0 [2026-10-28 12:01:15,815] INFO : Plotted jets +rank0 [2026-10-28 12:01:16,906] INFO : Plotted jet ratio +rank0 [2026-10-28 12:01:31,091] INFO : Step 15100/50000 rank0 | Train Loss: 9.5909 | Train Loss Unweighted: 3.2524 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.9163 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1391 | Regression_eta: 0.0098 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8264 | Classification_binary: 2.9163 | Regression_cos_phi: 0.5772 | Regression_energy: 1.8724 | Regression_eta: 0.4116 | Regression_pt: 2.0552 | Regression_sin_phi: 0.9319 | LR: 8.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3683s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1221 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3981.8 | uss_MB: 3501.1 | child_rss_MB: 3213.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:31,305] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:45,033] INFO : Step 15200/50000 rank0 | Train Loss: 9.4751 | Train Loss Unweighted: 3.2279 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.8864 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1447 | Regression_eta: 0.0109 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8325 | Classification_binary: 2.8864 | Regression_cos_phi: 0.5447 | Regression_energy: 1.9476 | Regression_eta: 0.4562 | Regression_pt: 2.0813 | Regression_sin_phi: 0.7265 | LR: 8.10e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2817s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0279 | max_backward: 0.0336 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.3849 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 3981.8 | uss_MB: 3499.9 | child_rss_MB: 3210.6 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:01:45,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:58,752] INFO : Step 15300/50000 rank0 | Train Loss: 10.3137 | Train Loss Unweighted: 3.2825 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.8944 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1653 | Regression_eta: 0.0124 | Regression_pt: 0.1638 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8737 | Classification_binary: 2.8944 | Regression_cos_phi: 0.5425 | Regression_energy: 2.2249 | Regression_eta: 0.5202 | Regression_pt: 2.3960 | Regression_sin_phi: 0.8619 | LR: 8.08e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0145 | max_loss: 0.0039 | max_backward: 0.0325 | max_optimizer_step: 0.1550 | max_tensorboard_logging: 0.2703 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3981.8 | uss_MB: 3501.2 | child_rss_MB: 3208.1 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:01:58,983] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:12,483] INFO : Step 15400/50000 rank0 | Train Loss: 9.5617 | Train Loss Unweighted: 3.2062 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8592 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1461 | Regression_eta: 0.0112 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.8592 | Regression_cos_phi: 0.4559 | Regression_energy: 1.9663 | Regression_eta: 0.4684 | Regression_pt: 2.1519 | Regression_sin_phi: 0.8700 | LR: 8.05e-04 | DataLoad Time: 0.0476s | Model Forward Time: 1.2243s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0333 | max_forward: 0.0170 | max_loss: 0.0034 | max_backward: 0.0375 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3844.9 | uss_MB: 3366.3 | child_rss_MB: 3208.0 | allocated_MB: 210.7 | reserved_MB: 2748.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:02:12,698] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:26,169] INFO : Step 15500/50000 rank0 | Train Loss: 9.2195 | Train Loss Unweighted: 3.1396 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.7910 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1507 | Regression_eta: 0.0099 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8364 | Classification_binary: 2.7910 | Regression_cos_phi: 0.4372 | Regression_energy: 2.0294 | Regression_eta: 0.4146 | Regression_pt: 2.1711 | Regression_sin_phi: 0.5398 | LR: 8.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2409s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0032 | max_backward: 0.1410 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3844.9 | uss_MB: 3364.7 | child_rss_MB: 3208.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:26,383] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:39,934] INFO : Step 15600/50000 rank0 | Train Loss: 9.8572 | Train Loss Unweighted: 3.3209 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.9505 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1611 | Regression_eta: 0.0098 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7359 | Classification_binary: 2.9505 | Regression_cos_phi: 0.4797 | Regression_energy: 2.1686 | Regression_eta: 0.4114 | Regression_pt: 2.3272 | Regression_sin_phi: 0.7838 | LR: 8.01e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2791s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0532 | max_forward: 0.0136 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1104 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3844.9 | uss_MB: 3365.0 | child_rss_MB: 3206.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:02:40,145] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:53,553] INFO : Step 15700/50000 rank0 | Train Loss: 9.8435 | Train Loss Unweighted: 3.3038 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.9473 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1515 | Regression_eta: 0.0123 | Regression_pt: 0.1486 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8204 | Classification_binary: 2.9473 | Regression_cos_phi: 0.5148 | Regression_energy: 2.0394 | Regression_eta: 0.5169 | Regression_pt: 2.1727 | Regression_sin_phi: 0.8321 | LR: 7.98e-04 | DataLoad Time: 0.0216s | Model Forward Time: 1.2396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.1372 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3844.9 | uss_MB: 3365.0 | child_rss_MB: 3207.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:02:53,766] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:07,231] INFO : Step 15800/50000 rank0 | Train Loss: 9.3396 | Train Loss Unweighted: 3.0977 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.7469 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1495 | Regression_eta: 0.0083 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8360 | Classification_binary: 2.7469 | Regression_cos_phi: 0.4908 | Regression_energy: 2.0130 | Regression_eta: 0.3499 | Regression_pt: 2.2000 | Regression_sin_phi: 0.7030 | LR: 7.96e-04 | DataLoad Time: 0.0491s | Model Forward Time: 1.3089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0785 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1154 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3844.9 | uss_MB: 3365.1 | child_rss_MB: 3210.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:07,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:21,288] INFO : Step 15900/50000 rank0 | Train Loss: 10.4359 | Train Loss Unweighted: 3.3478 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.9619 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1626 | Regression_eta: 0.0126 | Regression_pt: 0.1634 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8246 | Classification_binary: 2.9619 | Regression_cos_phi: 0.5949 | Regression_energy: 2.1896 | Regression_eta: 0.5271 | Regression_pt: 2.3903 | Regression_sin_phi: 0.9474 | LR: 7.94e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2724s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0616 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.3400 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3844.9 | uss_MB: 3365.1 | child_rss_MB: 3210.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:21,500] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:34,864] INFO : Step 16000/50000 rank0 | Train Loss: 9.8188 | Train Loss Unweighted: 3.1922 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.8361 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1482 | Regression_eta: 0.0130 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7992 | Classification_binary: 2.8361 | Regression_cos_phi: 0.6768 | Regression_energy: 1.9949 | Regression_eta: 0.5435 | Regression_pt: 2.1790 | Regression_sin_phi: 0.7895 | LR: 7.91e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2315s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0324 | max_optimizer_step: 0.2126 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3844.5 | uss_MB: 3364.4 | child_rss_MB: 3211.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:03:35,090] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:35,107] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 12:03:35,188] INFO : Running validation on rank0 +rank0 [2026-10-28 12:03:35,188] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:03:35,188] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:03:36,363] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 12:03:36,509] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:03:36,642] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:03:36,717] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:03:36,785] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:03:36,853] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:03:36,923] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:03:36,993] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:03:37,106] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:03:37,233] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:03:37,355] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:03:37,428] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:03:37,497] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:03:37,566] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:03:37,636] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:03:37,705] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:03:37,807] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:03:37,813] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:03:37,816] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:03:37,819] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:03:37,820] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:03:37,823] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:03:37,826] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:03:37,829] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:03:37,832] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:03:37,833] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:03:37,836] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:03:37,838] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:03:37,840] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:03:37,843] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:03:37,845] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:03:37,848] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:03:43,668] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.8188 | Valid Loss=9.5773 | Train Loss Unweighted=3.1922 | Valid Loss Unweighted=3.1936 | Stale=0 | ETA=108.7m +rank0 [2026-10-28 12:03:43,677] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:43,739] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:03:43,743] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:43,901] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:45,343] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:45,912] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:46,523] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:47,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:47,178] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:47,181] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:47,236] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:03:47,239] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:47,334] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:48,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:49,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:49,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:50,522] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:50,632] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:50,636] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:50,689] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:03:50,691] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:03:50,783] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:52,158] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:52,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:53,195] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:53,701] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:53,818] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:03:53,826] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:54,077] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:54,532] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:55,971] INFO : Plotted jets +rank0 [2026-10-28 12:03:57,058] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:57,091] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:57,316] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:57,695] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:59,511] INFO : Plotted jets +rank0 [2026-10-28 12:04:00,625] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:00,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 12:04:00,860] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 12:04:01,277] INFO : Plotted number of elements +rank0 [2026-10-28 12:04:02,674] INFO : Plotted jets +rank0 [2026-10-28 12:04:03,773] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:17,442] INFO : Step 16100/50000 rank0 | Train Loss: 9.6267 | Train Loss Unweighted: 3.2122 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8726 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1441 | Regression_eta: 0.0126 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7871 | Classification_binary: 2.8726 | Regression_cos_phi: 0.5837 | Regression_energy: 1.9395 | Regression_eta: 0.5275 | Regression_pt: 2.0125 | Regression_sin_phi: 0.9037 | LR: 7.89e-04 | DataLoad Time: 0.0824s | Model Forward Time: 1.2839s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0808 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1128 | max_tensorboard_logging: 0.3327 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3215.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:17,655] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:30,996] INFO : Step 16200/50000 rank0 | Train Loss: 10.2974 | Train Loss Unweighted: 3.2839 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8981 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1617 | Regression_eta: 0.0129 | Regression_pt: 0.1648 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8475 | Classification_binary: 2.8981 | Regression_cos_phi: 0.6017 | Regression_energy: 2.1765 | Regression_eta: 0.5411 | Regression_pt: 2.4106 | Regression_sin_phi: 0.8221 | LR: 7.86e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2193s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1821 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3209.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:31,226] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:44,709] INFO : Step 16300/50000 rank0 | Train Loss: 8.7498 | Train Loss Unweighted: 3.0414 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7248 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1342 | Regression_eta: 0.0103 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.7248 | Regression_cos_phi: 0.4333 | Regression_energy: 1.8071 | Regression_eta: 0.4301 | Regression_pt: 1.9544 | Regression_sin_phi: 0.6353 | LR: 7.84e-04 | DataLoad Time: 0.0013s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0588 | max_loss: 0.0031 | max_backward: 0.0324 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.3720 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4043.6 | uss_MB: 3566.1 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:04:44,923] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:58,296] INFO : Step 16400/50000 rank0 | Train Loss: 9.2254 | Train Loss Unweighted: 3.1327 | Train Loss Components: Classification: 0.0257 | Classification_binary: 2.7997 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1396 | Regression_eta: 0.0123 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8719 | Classification_binary: 2.7997 | Regression_cos_phi: 0.5175 | Regression_energy: 1.8789 | Regression_eta: 0.5172 | Regression_pt: 2.0196 | Regression_sin_phi: 0.6206 | LR: 7.81e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2245s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.2121 | max_tensorboard_logging: 0.2720 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:04:58,519] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:11,668] INFO : Step 16500/50000 rank0 | Train Loss: 9.6383 | Train Loss Unweighted: 3.2608 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9035 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1520 | Regression_eta: 0.0109 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7572 | Classification_binary: 2.9035 | Regression_cos_phi: 0.5213 | Regression_energy: 2.0467 | Regression_eta: 0.4570 | Regression_pt: 2.2459 | Regression_sin_phi: 0.7066 | LR: 7.79e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2564s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.1372 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:11,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:25,269] INFO : Step 16600/50000 rank0 | Train Loss: 9.0491 | Train Loss Unweighted: 3.1575 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.8307 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1384 | Regression_eta: 0.0069 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8124 | Classification_binary: 2.8307 | Regression_cos_phi: 0.5092 | Regression_energy: 1.8633 | Regression_eta: 0.2907 | Regression_pt: 2.0368 | Regression_sin_phi: 0.7060 | LR: 7.77e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0334 | max_loss: 0.0031 | max_backward: 0.0335 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3918.6 | uss_MB: 3438.2 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:25,483] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:38,959] INFO : Step 16700/50000 rank0 | Train Loss: 9.2943 | Train Loss Unweighted: 3.1391 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8098 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1372 | Regression_eta: 0.0109 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8059 | Classification_binary: 2.8098 | Regression_cos_phi: 0.5442 | Regression_energy: 1.8465 | Regression_eta: 0.4557 | Regression_pt: 2.0053 | Regression_sin_phi: 0.8269 | LR: 7.74e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0033 | max_backward: 0.1390 | max_optimizer_step: 0.1096 | max_tensorboard_logging: 0.2798 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3918.6 | uss_MB: 3438.3 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:05:39,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:52,457] INFO : Step 16800/50000 rank0 | Train Loss: 10.1016 | Train Loss Unweighted: 3.2139 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8307 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1632 | Regression_eta: 0.0113 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8491 | Classification_binary: 2.8307 | Regression_cos_phi: 0.5265 | Regression_energy: 2.1965 | Regression_eta: 0.4753 | Regression_pt: 2.3913 | Regression_sin_phi: 0.8322 | LR: 7.72e-04 | DataLoad Time: 0.0466s | Model Forward Time: 1.3442s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0165 | max_loss: 0.0921 | max_backward: 0.0339 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3918.6 | uss_MB: 3437.1 | child_rss_MB: 3204.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:05:52,670] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:05,837] INFO : Step 16900/50000 rank0 | Train Loss: 9.5274 | Train Loss Unweighted: 3.1959 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.8448 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1498 | Regression_eta: 0.0099 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7722 | Classification_binary: 2.8448 | Regression_cos_phi: 0.5667 | Regression_energy: 2.0165 | Regression_eta: 0.4133 | Regression_pt: 2.1786 | Regression_sin_phi: 0.7352 | LR: 7.69e-04 | DataLoad Time: 0.0725s | Model Forward Time: 1.2738s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0582 | max_loss: 0.0030 | max_backward: 0.0330 | max_optimizer_step: 0.1121 | max_tensorboard_logging: 0.3729 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3918.6 | uss_MB: 3438.4 | child_rss_MB: 3207.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:06:06,048] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:19,113] INFO : Step 17000/50000 rank0 | Train Loss: 10.1798 | Train Loss Unweighted: 3.2488 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.8548 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1678 | Regression_eta: 0.0122 | Regression_pt: 0.1724 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7508 | Classification_binary: 2.8548 | Regression_cos_phi: 0.5539 | Regression_energy: 2.2594 | Regression_eta: 0.5107 | Regression_pt: 2.5214 | Regression_sin_phi: 0.7286 | LR: 7.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3224s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0861 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3709 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3917.1 | uss_MB: 3438.3 | child_rss_MB: 3204.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:06:19,328] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:19,346] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 12:06:19,516] INFO : Running validation on rank0 +rank0 [2026-10-28 12:06:19,516] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:06:19,516] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:06:20,586] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 12:06:20,729] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:06:20,860] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:06:20,936] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:06:21,006] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:06:21,073] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:06:21,142] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:06:21,211] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:06:21,322] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:06:21,446] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:06:21,567] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:06:21,639] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:06:21,708] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:06:21,776] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:06:21,844] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:06:21,913] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:06:22,016] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:06:22,022] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:06:22,025] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:06:22,028] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:06:22,029] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:06:22,032] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:06:22,035] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:06:22,037] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:06:22,040] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:06:22,041] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:06:22,044] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:06:22,046] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:06:22,049] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:06:22,050] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:06:22,053] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:06:22,056] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:06:27,831] INFO : VALIDATION | Step=17000/50000 | Train Loss=10.1798 | Valid Loss=9.4903 | Train Loss Unweighted=3.2488 | Valid Loss Unweighted=3.1783 | Stale=0 | ETA=104.6m +rank0 [2026-10-28 12:06:27,841] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:27,902] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:06:27,906] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:28,065] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:29,539] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:30,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:30,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:31,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:31,423] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:06:31,427] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:31,483] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:06:31,487] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:31,576] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:33,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:33,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:34,132] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:34,661] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:34,753] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:06:34,756] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:06:34,811] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:06:34,814] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:06:34,974] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:06:36,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:06:36,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:06:37,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:06:37,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:06:38,030] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:06:38,038] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:38,287] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:06:38,738] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:40,175] INFO : Plotted jets +rank0 [2026-10-28 12:06:41,257] INFO : Plotted jet ratio +rank0 [2026-10-28 12:06:41,298] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:41,521] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:06:41,898] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:43,714] INFO : Plotted jets +rank0 [2026-10-28 12:06:44,822] INFO : Plotted jet ratio +rank0 [2026-10-28 12:06:44,855] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 12:06:45,055] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 12:06:45,470] INFO : Plotted number of elements +rank0 [2026-10-28 12:06:46,862] INFO : Plotted jets +rank0 [2026-10-28 12:06:47,954] INFO : Plotted jet ratio +rank0 [2026-10-28 12:07:01,580] INFO : Step 17100/50000 rank0 | Train Loss: 9.4942 | Train Loss Unweighted: 3.3069 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9519 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1540 | Regression_eta: 0.0099 | Regression_pt: 0.1525 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7901 | Classification_binary: 2.9519 | Regression_cos_phi: 0.4056 | Regression_energy: 2.0728 | Regression_eta: 0.4130 | Regression_pt: 2.2306 | Regression_sin_phi: 0.6302 | LR: 7.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2585s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.1432 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4112.9 | uss_MB: 3639.3 | child_rss_MB: 3202.8 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:07:01,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:14,842] INFO : Step 17200/50000 rank0 | Train Loss: 9.3799 | Train Loss Unweighted: 3.2388 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9048 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1404 | Regression_eta: 0.0124 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7911 | Classification_binary: 2.9048 | Regression_cos_phi: 0.5194 | Regression_energy: 1.8905 | Regression_eta: 0.5204 | Regression_pt: 2.0361 | Regression_sin_phi: 0.7176 | LR: 7.62e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.2594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0039 | max_backward: 0.1396 | max_optimizer_step: 0.1140 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4112.9 | uss_MB: 3637.3 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:07:15,057] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:28,203] INFO : Step 17300/50000 rank0 | Train Loss: 9.6138 | Train Loss Unweighted: 3.2544 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8936 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1546 | Regression_eta: 0.0119 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8044 | Classification_binary: 2.8936 | Regression_cos_phi: 0.4122 | Regression_energy: 2.0814 | Regression_eta: 0.4992 | Regression_pt: 2.2608 | Regression_sin_phi: 0.6621 | LR: 7.59e-04 | DataLoad Time: 0.0528s | Model Forward Time: 1.2566s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0031 | max_backward: 0.1338 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2753 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4112.9 | uss_MB: 3637.4 | child_rss_MB: 3202.8 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:07:28,418] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:41,333] INFO : Step 17400/50000 rank0 | Train Loss: 9.5478 | Train Loss Unweighted: 3.2325 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.8744 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1560 | Regression_eta: 0.0089 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7050 | Classification_binary: 2.8744 | Regression_cos_phi: 0.5162 | Regression_energy: 2.1005 | Regression_eta: 0.3744 | Regression_pt: 2.2473 | Regression_sin_phi: 0.7300 | LR: 7.57e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1938s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0140 | max_loss: 0.0030 | max_backward: 0.0324 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.2958 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3990.5 | uss_MB: 3514.0 | child_rss_MB: 3201.7 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:07:41,547] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:54,772] INFO : Step 17500/50000 rank0 | Train Loss: 8.8813 | Train Loss Unweighted: 3.0143 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6815 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1427 | Regression_eta: 0.0088 | Regression_pt: 0.1427 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7668 | Classification_binary: 2.6815 | Regression_cos_phi: 0.4575 | Regression_energy: 1.9214 | Regression_eta: 0.3684 | Regression_pt: 2.0870 | Regression_sin_phi: 0.5988 | LR: 7.54e-04 | DataLoad Time: 0.0263s | Model Forward Time: 1.1540s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0343 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3991.5 | uss_MB: 3514.6 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:07:55,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:08,167] INFO : Step 17600/50000 rank0 | Train Loss: 9.6255 | Train Loss Unweighted: 3.0583 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.7203 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1400 | Regression_eta: 0.0154 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7703 | Classification_binary: 2.7203 | Regression_cos_phi: 0.6761 | Regression_energy: 1.8846 | Regression_eta: 0.6474 | Regression_pt: 1.9807 | Regression_sin_phi: 0.9462 | LR: 7.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0148 | max_loss: 0.0033 | max_backward: 0.0888 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3991.5 | uss_MB: 3514.7 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:08:08,381] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:21,472] INFO : Step 17700/50000 rank0 | Train Loss: 9.1505 | Train Loss Unweighted: 3.1639 | Train Loss Components: Classification: 0.0251 | Classification_binary: 2.8279 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1425 | Regression_eta: 0.0086 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8532 | Classification_binary: 2.8279 | Regression_cos_phi: 0.4143 | Regression_energy: 1.9181 | Regression_eta: 0.3606 | Regression_pt: 2.1020 | Regression_sin_phi: 0.6744 | LR: 7.49e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2296s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0189 | max_loss: 0.0032 | max_backward: 0.0954 | max_optimizer_step: 0.1129 | max_tensorboard_logging: 0.2822 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3992.5 | uss_MB: 3517.2 | child_rss_MB: 3200.4 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:08:21,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:34,676] INFO : Step 17800/50000 rank0 | Train Loss: 8.9580 | Train Loss Unweighted: 2.9660 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6336 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1427 | Regression_eta: 0.0132 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7675 | Classification_binary: 2.6336 | Regression_cos_phi: 0.4313 | Regression_energy: 1.9214 | Regression_eta: 0.5524 | Regression_pt: 2.0167 | Regression_sin_phi: 0.6351 | LR: 7.47e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.1673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3992.5 | uss_MB: 3515.2 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:08:34,905] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:48,218] INFO : Step 17900/50000 rank0 | Train Loss: 8.8526 | Train Loss Unweighted: 3.0753 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7497 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1396 | Regression_eta: 0.0079 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7394 | Classification_binary: 2.7497 | Regression_cos_phi: 0.4273 | Regression_energy: 1.8789 | Regression_eta: 0.3309 | Regression_pt: 2.0456 | Regression_sin_phi: 0.6808 | LR: 7.44e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.1107 | max_backward: 0.0334 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3663 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3992.5 | uss_MB: 3514.1 | child_rss_MB: 3197.3 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:08:48,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,518] INFO : Step 18000/50000 rank0 | Train Loss: 8.8607 | Train Loss Unweighted: 2.9714 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6479 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1377 | Regression_eta: 0.0092 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7005 | Classification_binary: 2.6479 | Regression_cos_phi: 0.5440 | Regression_energy: 1.8541 | Regression_eta: 0.3863 | Regression_pt: 1.9991 | Regression_sin_phi: 0.7288 | LR: 7.42e-04 | DataLoad Time: 0.0174s | Model Forward Time: 1.3215s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0666 | max_backward: 0.0337 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3774 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3991.9 | uss_MB: 3512.9 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:01,728] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:01,745] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 12:09:01,825] INFO : Running validation on rank0 +rank0 [2026-10-28 12:09:01,825] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:09:01,825] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:09:02,909] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 12:09:03,054] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:09:03,185] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:09:03,260] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:09:03,328] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:09:03,397] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:09:03,468] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:09:03,539] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:09:03,650] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:09:03,774] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:09:03,896] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:09:03,967] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:09:04,037] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:09:04,104] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:09:04,174] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:09:04,244] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:09:04,348] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:09:04,354] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:09:04,357] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:09:04,358] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:09:04,361] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:09:04,363] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:09:04,364] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:09:04,367] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:09:04,370] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:09:04,373] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:09:04,374] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:09:04,377] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:09:04,379] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:09:04,380] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:09:04,383] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:09:04,385] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:09:10,200] INFO : VALIDATION | Step=18000/50000 | Train Loss=8.8607 | Valid Loss=9.4343 | Train Loss Unweighted=2.9714 | Valid Loss Unweighted=3.1684 | Stale=0 | ETA=100.6m +rank0 [2026-10-28 12:09:10,209] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:10,270] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:09:10,274] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:10,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:11,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:12,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:12,984] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:13,575] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:13,686] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:13,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:13,745] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:09:13,755] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:13,914] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:15,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:15,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:16,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:17,069] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:17,177] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:17,180] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:09:17,235] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:09:17,250] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:09:17,409] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:09:18,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:09:19,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:09:19,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:09:20,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:09:20,525] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:09:20,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:20,780] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:09:21,235] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:22,672] INFO : Plotted jets +rank0 [2026-10-28 12:09:23,755] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:23,787] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:24,010] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:09:24,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:26,219] INFO : Plotted jets +rank0 [2026-10-28 12:09:27,329] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:27,362] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 12:09:27,561] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 12:09:27,979] INFO : Plotted number of elements +rank0 [2026-10-28 12:09:29,375] INFO : Plotted jets +rank0 [2026-10-28 12:09:30,470] INFO : Plotted jet ratio +rank0 [2026-10-28 12:09:43,407] INFO : Step 18100/50000 rank0 | Train Loss: 9.9364 | Train Loss Unweighted: 3.1389 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7653 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1598 | Regression_eta: 0.0134 | Regression_pt: 0.1568 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7614 | Classification_binary: 2.7653 | Regression_cos_phi: 0.6029 | Regression_energy: 2.1516 | Regression_eta: 0.5602 | Regression_pt: 2.2930 | Regression_sin_phi: 0.8019 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0151 | max_loss: 0.0033 | max_backward: 0.0910 | max_optimizer_step: 0.1168 | max_tensorboard_logging: 0.2707 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4188.7 | uss_MB: 3711.5 | child_rss_MB: 3205.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:09:43,621] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:56,388] INFO : Step 18200/50000 rank0 | Train Loss: 9.0530 | Train Loss Unweighted: 3.2237 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.8909 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1443 | Regression_eta: 0.0088 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7586 | Classification_binary: 2.8909 | Regression_cos_phi: 0.4059 | Regression_energy: 1.9427 | Regression_eta: 0.3678 | Regression_pt: 2.0795 | Regression_sin_phi: 0.6076 | LR: 7.36e-04 | DataLoad Time: 0.0201s | Model Forward Time: 1.2574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0442 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.3196 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4188.7 | uss_MB: 3711.5 | child_rss_MB: 3203.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:09:56,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:09,669] INFO : Step 18300/50000 rank0 | Train Loss: 9.5721 | Train Loss Unweighted: 3.1719 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.8116 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1537 | Regression_eta: 0.0112 | Regression_pt: 0.1549 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7762 | Classification_binary: 2.8116 | Regression_cos_phi: 0.4728 | Regression_energy: 2.0694 | Regression_eta: 0.4709 | Regression_pt: 2.2647 | Regression_sin_phi: 0.7065 | LR: 7.34e-04 | DataLoad Time: 0.0414s | Model Forward Time: 1.3533s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1027 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3736 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4188.7 | uss_MB: 3711.6 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:10:09,882] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:22,699] INFO : Step 18400/50000 rank0 | Train Loss: 8.8849 | Train Loss Unweighted: 3.0690 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7434 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1379 | Regression_eta: 0.0087 | Regression_pt: 0.1401 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7510 | Classification_binary: 2.7434 | Regression_cos_phi: 0.4559 | Regression_energy: 1.8563 | Regression_eta: 0.3640 | Regression_pt: 2.0494 | Regression_sin_phi: 0.6650 | LR: 7.31e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.1984s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0033 | max_backward: 0.0603 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.2910 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4066.5 | uss_MB: 3587.8 | child_rss_MB: 3220.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:10:22,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:35,352] INFO : Step 18500/50000 rank0 | Train Loss: 9.3257 | Train Loss Unweighted: 3.0952 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.7397 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1550 | Regression_eta: 0.0093 | Regression_pt: 0.1524 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.7397 | Regression_cos_phi: 0.4608 | Regression_energy: 2.0865 | Regression_eta: 0.3902 | Regression_pt: 2.2286 | Regression_sin_phi: 0.6868 | LR: 7.29e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.1864s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0552 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.2791 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4066.5 | uss_MB: 3588.6 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:10:35,566] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:48,600] INFO : Step 18600/50000 rank0 | Train Loss: 10.0828 | Train Loss Unweighted: 3.1080 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7266 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1625 | Regression_eta: 0.0116 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7937 | Classification_binary: 2.7266 | Regression_cos_phi: 0.6805 | Regression_energy: 2.1880 | Regression_eta: 0.4868 | Regression_pt: 2.3474 | Regression_sin_phi: 0.8598 | LR: 7.26e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2675s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0164 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.2119 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4066.5 | uss_MB: 3588.8 | child_rss_MB: 3202.2 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:10:48,828] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:01,888] INFO : Step 18700/50000 rank0 | Train Loss: 9.3771 | Train Loss Unweighted: 3.1947 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8605 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1403 | Regression_eta: 0.0113 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7263 | Classification_binary: 2.8605 | Regression_cos_phi: 0.6075 | Regression_energy: 1.8888 | Regression_eta: 0.4715 | Regression_pt: 2.0525 | Regression_sin_phi: 0.7700 | LR: 7.24e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.2469s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1937 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.5 | uss_MB: 3588.9 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:02,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 123.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:14,939] INFO : Step 18800/50000 rank0 | Train Loss: 9.3665 | Train Loss Unweighted: 3.1102 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7613 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1485 | Regression_eta: 0.0097 | Regression_pt: 0.1490 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7548 | Classification_binary: 2.7613 | Regression_cos_phi: 0.5850 | Regression_energy: 1.9993 | Regression_eta: 0.4084 | Regression_pt: 2.1793 | Regression_sin_phi: 0.6785 | LR: 7.21e-04 | DataLoad Time: 0.0830s | Model Forward Time: 1.2769s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0635 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3269 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4066.5 | uss_MB: 3588.9 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:11:15,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:27,792] INFO : Step 18900/50000 rank0 | Train Loss: 9.6127 | Train Loss Unweighted: 3.1807 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.8064 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1628 | Regression_eta: 0.0083 | Regression_pt: 0.1633 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8024 | Classification_binary: 2.8064 | Regression_cos_phi: 0.4683 | Regression_energy: 2.1915 | Regression_eta: 0.3492 | Regression_pt: 2.3881 | Regression_sin_phi: 0.6068 | LR: 7.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1600 | max_tensorboard_logging: 0.2809 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4066.5 | uss_MB: 3589.0 | child_rss_MB: 3201.0 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:28,024] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:40,466] INFO : Step 19000/50000 rank0 | Train Loss: 8.7202 | Train Loss Unweighted: 3.0424 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7234 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1366 | Regression_eta: 0.0099 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7017 | Classification_binary: 2.7234 | Regression_cos_phi: 0.4923 | Regression_energy: 1.8383 | Regression_eta: 0.4131 | Regression_pt: 1.9853 | Regression_sin_phi: 0.5662 | LR: 7.16e-04 | DataLoad Time: 0.0238s | Model Forward Time: 1.2594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0167 | max_loss: 0.0035 | max_backward: 0.1196 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.2857 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4066.3 | uss_MB: 3588.0 | child_rss_MB: 3224.9 | allocated_MB: 207.0 | reserved_MB: 2750.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:11:40,678] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:11:40,696] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 12:11:40,776] INFO : Running validation on rank0 +rank0 [2026-10-28 12:11:40,776] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:11:40,776] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:11:41,958] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 12:11:42,101] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:11:42,234] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:11:42,307] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:11:42,375] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:11:42,443] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:11:42,513] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:11:42,582] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:11:42,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:11:42,819] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:11:42,942] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:11:43,014] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:11:43,083] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:11:43,152] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:11:43,221] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:11:43,291] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:11:43,396] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:11:43,402] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:11:43,405] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:11:43,408] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:11:43,409] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:11:43,412] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:11:43,415] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:11:43,416] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:11:43,418] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:11:43,421] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:11:43,422] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:11:43,424] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:11:43,427] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:11:43,428] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:11:43,431] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:11:43,434] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:11:49,368] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.7202 | Valid Loss=9.4453 | Train Loss Unweighted=3.0424 | Valid Loss Unweighted=3.1741 | Stale=1 | ETA=96.7m +rank0 [2026-10-28 12:11:49,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:49,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:11:49,442] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:11:49,605] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:51,049] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:51,609] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:52,175] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:52,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:52,893] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:52,896] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:52,952] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:11:52,955] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:11:53,123] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:54,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:55,102] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:55,643] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:56,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:56,388] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:56,391] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:11:56,444] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:11:56,447] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:11:56,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:11:58,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:11:58,632] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:11:59,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:11:59,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:11:59,888] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:11:59,896] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:12:00,146] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:12:00,596] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:02,045] INFO : Plotted jets +rank0 [2026-10-28 12:12:03,139] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:03,171] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:03,391] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:12:03,772] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:05,618] INFO : Plotted jets +rank0 [2026-10-28 12:12:06,740] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:06,773] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 12:12:06,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 12:12:07,395] INFO : Plotted number of elements +rank0 [2026-10-28 12:12:08,797] INFO : Plotted jets +rank0 [2026-10-28 12:12:09,900] INFO : Plotted jet ratio +rank0 [2026-10-28 12:12:22,827] INFO : Step 19100/50000 rank0 | Train Loss: 9.4328 | Train Loss Unweighted: 3.2166 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.8717 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1465 | Regression_eta: 0.0122 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7565 | Classification_binary: 2.8717 | Regression_cos_phi: 0.5552 | Regression_energy: 1.9720 | Regression_eta: 0.5094 | Regression_pt: 2.1309 | Regression_sin_phi: 0.6370 | LR: 7.13e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3042s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0452 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3712 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4270.0 | uss_MB: 3789.6 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:12:23,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:35,512] INFO : Step 19200/50000 rank0 | Train Loss: 9.4367 | Train Loss Unweighted: 3.0768 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.7381 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1409 | Regression_eta: 0.0129 | Regression_pt: 0.1403 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8070 | Classification_binary: 2.7381 | Regression_cos_phi: 0.5426 | Regression_energy: 1.8970 | Regression_eta: 0.5399 | Regression_pt: 2.0523 | Regression_sin_phi: 0.8597 | LR: 7.10e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3018s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0951 | max_forward: 0.0142 | max_loss: 0.0034 | max_backward: 0.0314 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.3159 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4270.0 | uss_MB: 3789.6 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:12:35,728] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:12:48,453] INFO : Step 19300/50000 rank0 | Train Loss: 9.1221 | Train Loss Unweighted: 3.0047 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.6678 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1413 | Regression_eta: 0.0105 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7306 | Classification_binary: 2.6678 | Regression_cos_phi: 0.5114 | Regression_energy: 1.9026 | Regression_eta: 0.4405 | Regression_pt: 2.1163 | Regression_sin_phi: 0.7529 | LR: 7.08e-04 | DataLoad Time: 0.0014s | Model Forward Time: 1.2735s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.2113 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4270.0 | uss_MB: 3789.7 | child_rss_MB: 3199.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:12:48,680] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:01,121] INFO : Step 19400/50000 rank0 | Train Loss: 8.5605 | Train Loss Unweighted: 3.0426 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.7370 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1290 | Regression_eta: 0.0105 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7124 | Classification_binary: 2.7370 | Regression_cos_phi: 0.3928 | Regression_energy: 1.7367 | Regression_eta: 0.4418 | Regression_pt: 1.8965 | Regression_sin_phi: 0.6433 | LR: 7.05e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.1373 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4270.0 | uss_MB: 3789.8 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:01,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:13,999] INFO : Step 19500/50000 rank0 | Train Loss: 10.2405 | Train Loss Unweighted: 3.1777 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7851 | Regression_cos_phi: 0.0122 | Regression_energy: 0.1684 | Regression_eta: 0.0106 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7583 | Classification_binary: 2.7851 | Regression_cos_phi: 0.6648 | Regression_energy: 2.2677 | Regression_eta: 0.4437 | Regression_pt: 2.4596 | Regression_sin_phi: 0.8612 | LR: 7.02e-04 | DataLoad Time: 0.0663s | Model Forward Time: 1.3521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1108 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1146 | max_tensorboard_logging: 0.3726 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4143.7 | uss_MB: 3663.2 | child_rss_MB: 3198.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:14,211] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:26,591] INFO : Step 19600/50000 rank0 | Train Loss: 9.0382 | Train Loss Unweighted: 2.9997 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6659 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1417 | Regression_eta: 0.0098 | Regression_pt: 0.1412 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7824 | Classification_binary: 2.6659 | Regression_cos_phi: 0.4995 | Regression_energy: 1.9076 | Regression_eta: 0.4112 | Regression_pt: 2.0644 | Regression_sin_phi: 0.7073 | LR: 7.00e-04 | DataLoad Time: 0.0732s | Model Forward Time: 1.3544s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1171 | max_loss: 0.0036 | max_backward: 0.0329 | max_optimizer_step: 0.1118 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:13:26,805] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:39,177] INFO : Step 19700/50000 rank0 | Train Loss: 8.5923 | Train Loss Unweighted: 2.8780 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5702 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1287 | Regression_eta: 0.0112 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6922 | Classification_binary: 2.5702 | Regression_cos_phi: 0.5372 | Regression_energy: 1.7331 | Regression_eta: 0.4700 | Regression_pt: 1.8823 | Regression_sin_phi: 0.7074 | LR: 6.97e-04 | DataLoad Time: 0.0472s | Model Forward Time: 1.3205s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0793 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3199.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:13:39,395] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:52,076] INFO : Step 19800/50000 rank0 | Train Loss: 9.1250 | Train Loss Unweighted: 2.9402 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.5984 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1453 | Regression_eta: 0.0118 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7823 | Classification_binary: 2.5984 | Regression_cos_phi: 0.5030 | Regression_energy: 1.9567 | Regression_eta: 0.4932 | Regression_pt: 2.1013 | Regression_sin_phi: 0.6901 | LR: 6.94e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1352 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3199.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:13:52,304] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:04,726] INFO : Step 19900/50000 rank0 | Train Loss: 9.8769 | Train Loss Unweighted: 3.0803 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7108 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1573 | Regression_eta: 0.0106 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7161 | Classification_binary: 2.7108 | Regression_cos_phi: 0.5994 | Regression_energy: 2.1177 | Regression_eta: 0.4462 | Regression_pt: 2.2940 | Regression_sin_phi: 0.9925 | LR: 6.92e-04 | DataLoad Time: 0.0143s | Model Forward Time: 1.1956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0558 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.2813 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4143.7 | uss_MB: 3663.8 | child_rss_MB: 3206.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:14:04,942] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:17,249] INFO : Step 20000/50000 rank0 | Train Loss: 9.1370 | Train Loss Unweighted: 3.1946 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.8668 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1393 | Regression_eta: 0.0101 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8382 | Classification_binary: 2.8668 | Regression_cos_phi: 0.4600 | Regression_energy: 1.8757 | Regression_eta: 0.4251 | Regression_pt: 1.9999 | Regression_sin_phi: 0.6713 | LR: 6.89e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0171 | max_loss: 0.0033 | max_backward: 0.0389 | max_optimizer_step: 0.2219 | max_tensorboard_logging: 0.2901 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4144.7 | uss_MB: 3666.4 | child_rss_MB: 3206.1 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:14:17,477] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:17,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 12:14:17,576] INFO : Running validation on rank0 +rank0 [2026-10-28 12:14:17,576] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:14:17,576] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:14:18,742] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 12:14:18,887] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:14:19,019] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:14:19,093] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:14:19,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:14:19,228] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:14:19,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:14:19,368] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:14:19,481] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:14:19,604] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:14:19,725] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:14:19,796] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:14:19,864] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:14:19,932] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:14:20,001] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:14:20,071] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:14:20,172] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:14:20,178] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:14:20,181] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:14:20,184] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:14:20,185] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:14:20,187] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:14:20,189] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:14:20,191] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:14:20,194] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:14:20,195] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:14:20,198] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:14:20,201] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:14:20,204] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:14:20,207] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:14:20,208] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:14:20,211] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:14:26,088] INFO : VALIDATION | Step=20000/50000 | Train Loss=9.1370 | Valid Loss=9.3028 | Train Loss Unweighted=3.1946 | Valid Loss Unweighted=3.1245 | Stale=0 | ETA=92.8m +rank0 [2026-10-28 12:14:26,097] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:26,159] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:14:26,163] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:26,330] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:27,716] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:28,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:28,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:29,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:29,686] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:14:29,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:29,744] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:14:29,747] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:29,911] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:31,318] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:31,902] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:32,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:33,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:33,209] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:14:33,212] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:14:33,266] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:14:33,269] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:14:33,435] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:14:34,850] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:14:35,367] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:14:35,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:14:36,459] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:14:36,590] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:14:36,597] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:36,847] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:14:37,302] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:38,743] INFO : Plotted jets +rank0 [2026-10-28 12:14:39,848] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:39,881] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:40,104] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:14:40,481] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:42,363] INFO : Plotted jets +rank0 [2026-10-28 12:14:43,485] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:43,519] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 12:14:43,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 12:14:44,140] INFO : Plotted number of elements +rank0 [2026-10-28 12:14:45,539] INFO : Plotted jets +rank0 [2026-10-28 12:14:46,642] INFO : Plotted jet ratio +rank0 [2026-10-28 12:14:59,276] INFO : Step 20100/50000 rank0 | Train Loss: 9.4646 | Train Loss Unweighted: 3.0714 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7355 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1364 | Regression_eta: 0.0213 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 2.7355 | Regression_cos_phi: 0.4837 | Regression_energy: 1.8365 | Regression_eta: 0.8923 | Regression_pt: 2.0015 | Regression_sin_phi: 0.7256 | LR: 6.86e-04 | DataLoad Time: 0.0015s | Model Forward Time: 1.3609s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0799 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4343.8 | uss_MB: 3864.9 | child_rss_MB: 3204.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:14:59,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:11,784] INFO : Step 20200/50000 rank0 | Train Loss: 8.9915 | Train Loss Unweighted: 3.0969 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.7853 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1306 | Regression_eta: 0.0109 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6943 | Classification_binary: 2.7853 | Regression_cos_phi: 0.5531 | Regression_energy: 1.7587 | Regression_eta: 0.4580 | Regression_pt: 1.8797 | Regression_sin_phi: 0.8625 | LR: 6.84e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4078s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0959 | max_backward: 0.0338 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3961 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4343.8 | uss_MB: 3864.9 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:15:11,994] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:24,592] INFO : Step 20300/50000 rank0 | Train Loss: 9.0444 | Train Loss Unweighted: 3.0223 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.6928 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1397 | Regression_eta: 0.0095 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7550 | Classification_binary: 2.6928 | Regression_cos_phi: 0.4464 | Regression_energy: 1.8804 | Regression_eta: 0.3971 | Regression_pt: 2.0394 | Regression_sin_phi: 0.8333 | LR: 6.81e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.2209s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1189 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 4343.8 | uss_MB: 3865.0 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:15:24,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:37,088] INFO : Step 20400/50000 rank0 | Train Loss: 9.0596 | Train Loss Unweighted: 2.9935 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.6669 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1368 | Regression_eta: 0.0117 | Regression_pt: 0.1369 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7242 | Classification_binary: 2.6669 | Regression_cos_phi: 0.5375 | Regression_energy: 1.8413 | Regression_eta: 0.4904 | Regression_pt: 2.0015 | Regression_sin_phi: 0.7978 | LR: 6.78e-04 | DataLoad Time: 0.0368s | Model Forward Time: 1.3194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0570 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.3290 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4343.8 | uss_MB: 3865.0 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:15:37,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:49,873] INFO : Step 20500/50000 rank0 | Train Loss: 8.6218 | Train Loss Unweighted: 3.0504 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7346 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1348 | Regression_eta: 0.0092 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7010 | Classification_binary: 2.7346 | Regression_cos_phi: 0.3867 | Regression_energy: 1.8145 | Regression_eta: 0.3875 | Regression_pt: 1.9940 | Regression_sin_phi: 0.6036 | LR: 6.76e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2567s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.2007 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4215.4 | uss_MB: 3736.5 | child_rss_MB: 3206.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:15:50,099] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:02,226] INFO : Step 20600/50000 rank0 | Train Loss: 9.1607 | Train Loss Unweighted: 3.0830 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7366 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1491 | Regression_eta: 0.0091 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7477 | Classification_binary: 2.7366 | Regression_cos_phi: 0.4726 | Regression_energy: 2.0069 | Regression_eta: 0.3830 | Regression_pt: 2.1872 | Regression_sin_phi: 0.6268 | LR: 6.73e-04 | DataLoad Time: 0.0446s | Model Forward Time: 1.2999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0670 | max_forward: 0.0189 | max_loss: 0.0034 | max_backward: 0.0389 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3235 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4216.4 | uss_MB: 3739.0 | child_rss_MB: 3205.9 | allocated_MB: 210.7 | reserved_MB: 2802.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:16:02,441] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:14,463] INFO : Step 20700/50000 rank0 | Train Loss: 10.0484 | Train Loss Unweighted: 3.1710 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.7894 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1622 | Regression_eta: 0.0113 | Regression_pt: 0.1632 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7878 | Classification_binary: 2.7894 | Regression_cos_phi: 0.6495 | Regression_energy: 2.1830 | Regression_eta: 0.4722 | Regression_pt: 2.3866 | Regression_sin_phi: 0.7799 | LR: 6.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2282s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0161 | max_loss: 0.0033 | max_backward: 0.0825 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4216.4 | uss_MB: 3737.0 | child_rss_MB: 3207.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:16:14,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:26,727] INFO : Step 20800/50000 rank0 | Train Loss: 9.8466 | Train Loss Unweighted: 3.1431 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.7624 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1640 | Regression_eta: 0.0121 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8033 | Classification_binary: 2.7624 | Regression_cos_phi: 0.5079 | Regression_energy: 2.2074 | Regression_eta: 0.5054 | Regression_pt: 2.3848 | Regression_sin_phi: 0.6754 | LR: 6.67e-04 | DataLoad Time: 0.0580s | Model Forward Time: 1.3640s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0729 | max_loss: 0.0035 | max_backward: 0.0335 | max_optimizer_step: 0.1180 | max_tensorboard_logging: 0.3737 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4216.4 | uss_MB: 3737.1 | child_rss_MB: 3210.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:16:26,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:38,924] INFO : Step 20900/50000 rank0 | Train Loss: 9.2803 | Train Loss Unweighted: 3.0268 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6909 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1425 | Regression_eta: 0.0117 | Regression_pt: 0.1386 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7879 | Classification_binary: 2.6909 | Regression_cos_phi: 0.4520 | Regression_energy: 1.9190 | Regression_eta: 0.4896 | Regression_pt: 2.0268 | Regression_sin_phi: 0.9140 | LR: 6.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2160s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0319 | max_optimizer_step: 0.1527 | max_tensorboard_logging: 0.2780 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4216.4 | uss_MB: 3737.2 | child_rss_MB: 3210.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:16:39,150] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:51,124] INFO : Step 21000/50000 rank0 | Train Loss: 9.1252 | Train Loss Unweighted: 3.1063 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.7745 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1405 | Regression_eta: 0.0103 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7180 | Classification_binary: 2.7745 | Regression_cos_phi: 0.4627 | Regression_energy: 1.8915 | Regression_eta: 0.4300 | Regression_pt: 2.0698 | Regression_sin_phi: 0.7786 | LR: 6.62e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3654s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0824 | max_backward: 0.0328 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3678 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4216.4 | uss_MB: 3736.1 | child_rss_MB: 3209.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:16:51,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:51,364] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 12:16:51,549] INFO : Running validation on rank0 +rank0 [2026-10-28 12:16:51,549] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:16:51,549] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:16:52,608] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 12:16:52,751] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:16:52,882] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:16:52,956] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:16:53,025] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:16:53,094] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:16:53,164] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:16:53,234] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:16:53,346] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:16:53,470] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:16:53,592] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:16:53,665] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:16:53,736] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:16:53,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:16:53,875] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:16:53,944] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:16:54,055] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:16:54,061] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:16:54,064] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:16:54,067] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:16:54,068] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:16:54,071] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:16:54,074] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:16:54,075] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:16:54,078] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:16:54,081] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:16:54,083] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:16:54,084] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:16:54,087] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:16:54,090] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:16:54,093] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:16:54,094] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:16:59,887] INFO : VALIDATION | Step=21000/50000 | Train Loss=9.1252 | Valid Loss=9.3027 | Train Loss Unweighted=3.1063 | Valid Loss Unweighted=3.0990 | Stale=0 | ETA=89.0m +rank0 [2026-10-28 12:16:59,897] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:16:59,960] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:16:59,963] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:00,126] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:01,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:02,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:02,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:03,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:03,434] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:03,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:03,494] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:17:03,497] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:03,590] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:05,006] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:05,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:06,081] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:06,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:06,820] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:06,823] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:06,878] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:17:06,881] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:17:07,044] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:08,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:08,868] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:09,486] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:09,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:10,122] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:17:10,129] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:10,380] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:10,836] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:12,279] INFO : Plotted jets +rank0 [2026-10-28 12:17:13,366] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:13,398] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:13,623] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:14,001] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:15,849] INFO : Plotted jets +rank0 [2026-10-28 12:17:16,965] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:16,999] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:17,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:17,642] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:19,041] INFO : Plotted jets +rank0 [2026-10-28 12:17:20,131] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:33,068] INFO : Step 21100/50000 rank0 | Train Loss: 9.5936 | Train Loss Unweighted: 3.2061 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.8439 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1565 | Regression_eta: 0.0081 | Regression_pt: 0.1571 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7401 | Classification_binary: 2.8439 | Regression_cos_phi: 0.4770 | Regression_energy: 2.1066 | Regression_eta: 0.3377 | Regression_pt: 2.2970 | Regression_sin_phi: 0.7913 | LR: 6.59e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.1956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0141 | max_loss: 0.0032 | max_backward: 0.0326 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4415.3 | uss_MB: 3934.8 | child_rss_MB: 3207.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:33,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:34,145] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 12:17:34,145] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 12:17:44,300] INFO : Step 21200/50000 rank0 | Train Loss: 9.0116 | Train Loss Unweighted: 3.0237 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6884 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1431 | Regression_eta: 0.0102 | Regression_pt: 0.1426 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7705 | Classification_binary: 2.6884 | Regression_cos_phi: 0.4648 | Regression_energy: 1.9264 | Regression_eta: 0.4290 | Regression_pt: 2.0858 | Regression_sin_phi: 0.6468 | LR: 6.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2618s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0031 | max_backward: 0.1190 | max_optimizer_step: 0.1153 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4453.5 | uss_MB: 3973.1 | child_rss_MB: 3228.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:44,512] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:55,180] INFO : Step 21300/50000 rank0 | Train Loss: 9.2374 | Train Loss Unweighted: 3.0757 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7334 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1470 | Regression_eta: 0.0108 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.7334 | Regression_cos_phi: 0.4271 | Regression_energy: 1.9792 | Regression_eta: 0.4537 | Regression_pt: 2.1026 | Regression_sin_phi: 0.7488 | LR: 6.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.1813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0137 | Memory: rss_MB: 4453.5 | uss_MB: 3973.1 | child_rss_MB: 3211.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:17:55,391] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:06,147] INFO : Step 21400/50000 rank0 | Train Loss: 9.2566 | Train Loss Unweighted: 3.0260 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6866 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1424 | Regression_eta: 0.0124 | Regression_pt: 0.1432 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7671 | Classification_binary: 2.6866 | Regression_cos_phi: 0.4784 | Regression_energy: 1.9167 | Regression_eta: 0.5188 | Regression_pt: 2.0948 | Regression_sin_phi: 0.7942 | LR: 6.51e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.2806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0331 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4453.5 | uss_MB: 3973.2 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:06,360] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:17,043] INFO : Step 21500/50000 rank0 | Train Loss: 8.8602 | Train Loss Unweighted: 3.0322 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.7125 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1361 | Regression_eta: 0.0107 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7281 | Classification_binary: 2.7125 | Regression_cos_phi: 0.4642 | Regression_energy: 1.8321 | Regression_eta: 0.4498 | Regression_pt: 1.9575 | Regression_sin_phi: 0.7160 | LR: 6.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2546s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0529 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.3174 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4286.0 | uss_MB: 3806.1 | child_rss_MB: 3243.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:17,258] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:28,094] INFO : Step 21600/50000 rank0 | Train Loss: 8.9754 | Train Loss Unweighted: 2.9932 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6670 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1374 | Regression_eta: 0.0094 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.6670 | Regression_cos_phi: 0.5391 | Regression_energy: 1.8494 | Regression_eta: 0.3932 | Regression_pt: 2.0281 | Regression_sin_phi: 0.7934 | LR: 6.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2913s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0031 | max_backward: 0.1367 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.2856 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4287.0 | uss_MB: 3807.1 | child_rss_MB: 3211.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:28,309] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:39,155] INFO : Step 21700/50000 rank0 | Train Loss: 8.6080 | Train Loss Unweighted: 2.9433 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6211 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1387 | Regression_eta: 0.0084 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6917 | Classification_binary: 2.6211 | Regression_cos_phi: 0.4209 | Regression_energy: 1.8674 | Regression_eta: 0.3504 | Regression_pt: 2.0353 | Regression_sin_phi: 0.6211 | LR: 6.43e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2494s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0157 | max_loss: 0.0030 | max_backward: 0.0343 | max_optimizer_step: 0.1487 | max_tensorboard_logging: 0.3293 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4287.0 | uss_MB: 3807.1 | child_rss_MB: 3210.9 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:18:39,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:18:50,021] INFO : Step 21800/50000 rank0 | Train Loss: 8.9894 | Train Loss Unweighted: 3.0161 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.6833 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1428 | Regression_eta: 0.0088 | Regression_pt: 0.1430 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6643 | Classification_binary: 2.6833 | Regression_cos_phi: 0.4654 | Regression_energy: 1.9226 | Regression_eta: 0.3671 | Regression_pt: 2.0910 | Regression_sin_phi: 0.7956 | LR: 6.40e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3319s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1020 | max_forward: 0.0163 | max_loss: 0.0030 | max_backward: 0.0333 | max_optimizer_step: 0.1151 | max_tensorboard_logging: 0.3375 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4287.0 | uss_MB: 3807.2 | child_rss_MB: 3214.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:18:50,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:01,103] INFO : Step 21900/50000 rank0 | Train Loss: 9.0555 | Train Loss Unweighted: 3.0828 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7568 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1385 | Regression_eta: 0.0126 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.7568 | Regression_cos_phi: 0.4308 | Regression_energy: 1.8645 | Regression_eta: 0.5273 | Regression_pt: 1.9831 | Regression_sin_phi: 0.7458 | LR: 6.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3044s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0782 | max_forward: 0.0164 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3238 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4287.0 | uss_MB: 3807.2 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:19:01,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:12,245] INFO : Step 22000/50000 rank0 | Train Loss: 9.7307 | Train Loss Unweighted: 3.1206 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.7446 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1626 | Regression_eta: 0.0106 | Regression_pt: 0.1615 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8043 | Classification_binary: 2.7446 | Regression_cos_phi: 0.4527 | Regression_energy: 2.1889 | Regression_eta: 0.4450 | Regression_pt: 2.3614 | Regression_sin_phi: 0.7338 | LR: 6.34e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3335s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1019 | max_forward: 0.0156 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.3273 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4287.0 | uss_MB: 3807.3 | child_rss_MB: 3217.6 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:19:12,459] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:12,502] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 12:19:12,583] INFO : Running validation on rank0 +rank0 [2026-10-28 12:19:12,583] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:19:12,583] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:19:13,674] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 12:19:13,817] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:19:13,948] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:19:14,023] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:19:14,092] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:19:14,160] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:19:14,230] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:19:14,300] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:19:14,411] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:19:14,536] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:19:14,658] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:19:14,732] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:19:14,802] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:19:14,871] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:19:14,940] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:19:15,018] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:19:15,138] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:19:15,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:19:15,147] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:19:15,150] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:19:15,151] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:19:15,154] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:19:15,157] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:19:15,159] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:19:15,160] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:19:15,163] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:19:15,164] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:19:15,167] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:19:15,169] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:19:15,170] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:19:15,173] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:19:15,176] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:19:20,990] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.7307 | Valid Loss=9.1295 | Train Loss Unweighted=3.1206 | Valid Loss Unweighted=3.0547 | Stale=0 | ETA=85.0m +rank0 [2026-10-28 12:19:21,002] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:21,067] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:19:21,071] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:21,237] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:22,604] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:23,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:23,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:24,298] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:24,424] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:24,428] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:24,485] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:19:24,487] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:24,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:25,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:26,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:27,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:27,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:27,868] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:27,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:19:27,925] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:19:27,928] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:19:28,039] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:19:29,449] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:19:29,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:19:30,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:19:31,071] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:19:31,208] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:19:31,216] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,468] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:19:31,921] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:33,364] INFO : Plotted jets +rank0 [2026-10-28 12:19:34,446] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:34,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:34,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:19:35,087] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:36,942] INFO : Plotted jets +rank0 [2026-10-28 12:19:38,050] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:38,082] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 12:19:38,284] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 12:19:38,701] INFO : Plotted number of elements +rank0 [2026-10-28 12:19:40,094] INFO : Plotted jets +rank0 [2026-10-28 12:19:41,184] INFO : Plotted jet ratio +rank0 [2026-10-28 12:19:52,415] INFO : Step 22100/50000 rank0 | Train Loss: 8.3414 | Train Loss Unweighted: 2.9058 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5959 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1329 | Regression_eta: 0.0083 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7125 | Classification_binary: 2.5959 | Regression_cos_phi: 0.3878 | Regression_energy: 1.7894 | Regression_eta: 0.3493 | Regression_pt: 1.9525 | Regression_sin_phi: 0.5540 | LR: 6.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3472s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.0372 | max_backward: 0.0339 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.3899 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4492.3 | uss_MB: 4013.6 | child_rss_MB: 3213.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:19:52,625] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:03,570] INFO : Step 22200/50000 rank0 | Train Loss: 8.7378 | Train Loss Unweighted: 2.8832 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.5703 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1304 | Regression_eta: 0.0101 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7389 | Classification_binary: 2.5703 | Regression_cos_phi: 0.4395 | Regression_energy: 1.7561 | Regression_eta: 0.4237 | Regression_pt: 1.9188 | Regression_sin_phi: 0.8905 | LR: 6.29e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2956s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0133 | max_backward: 0.0328 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4492.3 | uss_MB: 4012.5 | child_rss_MB: 3216.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:03,786] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:14,615] INFO : Step 22300/50000 rank0 | Train Loss: 8.9905 | Train Loss Unweighted: 3.0347 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6960 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1459 | Regression_eta: 0.0102 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7162 | Classification_binary: 2.6960 | Regression_cos_phi: 0.4007 | Regression_energy: 1.9642 | Regression_eta: 0.4273 | Regression_pt: 2.1343 | Regression_sin_phi: 0.6520 | LR: 6.26e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2323s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0567 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4492.3 | uss_MB: 4013.8 | child_rss_MB: 3240.9 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:14,831] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:25,877] INFO : Step 22400/50000 rank0 | Train Loss: 9.1353 | Train Loss Unweighted: 3.1363 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7949 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1482 | Regression_eta: 0.0085 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7663 | Classification_binary: 2.7949 | Regression_cos_phi: 0.4483 | Regression_energy: 1.9955 | Regression_eta: 0.3559 | Regression_pt: 2.1320 | Regression_sin_phi: 0.6424 | LR: 6.23e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3818s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0769 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3242 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4492.3 | uss_MB: 4012.7 | child_rss_MB: 3215.7 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:20:26,091] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:37,033] INFO : Step 22500/50000 rank0 | Train Loss: 8.5704 | Train Loss Unweighted: 3.0317 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.7279 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1295 | Regression_eta: 0.0092 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7336 | Classification_binary: 2.7279 | Regression_cos_phi: 0.4771 | Regression_energy: 1.7436 | Regression_eta: 0.3867 | Regression_pt: 1.8486 | Regression_sin_phi: 0.6527 | LR: 6.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0329 | max_backward: 0.0337 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.4008 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4363.9 | uss_MB: 3885.1 | child_rss_MB: 3213.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:37,247] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:48,131] INFO : Step 22600/50000 rank0 | Train Loss: 8.5830 | Train Loss Unweighted: 2.9290 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6138 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1350 | Regression_eta: 0.0099 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7174 | Classification_binary: 2.6138 | Regression_cos_phi: 0.4004 | Regression_energy: 1.8177 | Regression_eta: 0.4132 | Regression_pt: 1.9506 | Regression_sin_phi: 0.6700 | LR: 6.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3149s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0505 | max_backward: 0.0335 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4363.9 | uss_MB: 3884.7 | child_rss_MB: 3217.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:20:48,344] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:58,968] INFO : Step 22700/50000 rank0 | Train Loss: 9.3718 | Train Loss Unweighted: 3.0961 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7625 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1388 | Regression_eta: 0.0112 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7644 | Classification_binary: 2.7625 | Regression_cos_phi: 0.5496 | Regression_energy: 1.8689 | Regression_eta: 0.4684 | Regression_pt: 2.0369 | Regression_sin_phi: 0.9211 | LR: 6.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2124s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1566 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4363.9 | uss_MB: 3886.0 | child_rss_MB: 3215.2 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:20:59,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:10,068] INFO : Step 22800/50000 rank0 | Train Loss: 8.5136 | Train Loss Unweighted: 2.9794 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6731 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1288 | Regression_eta: 0.0078 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7835 | Classification_binary: 2.6731 | Regression_cos_phi: 0.4744 | Regression_energy: 1.7343 | Regression_eta: 0.3285 | Regression_pt: 1.9018 | Regression_sin_phi: 0.6180 | LR: 6.12e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1674 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4364.9 | uss_MB: 3886.4 | child_rss_MB: 3212.1 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:21:10,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:20,981] INFO : Step 22900/50000 rank0 | Train Loss: 8.9764 | Train Loss Unweighted: 2.9652 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.6183 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1516 | Regression_eta: 0.0113 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6862 | Classification_binary: 2.6183 | Regression_cos_phi: 0.4573 | Regression_energy: 2.0403 | Regression_eta: 0.4715 | Regression_pt: 2.1751 | Regression_sin_phi: 0.5277 | LR: 6.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.1750s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4364.9 | uss_MB: 3886.5 | child_rss_MB: 3212.5 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:21:21,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:32,021] INFO : Step 23000/50000 rank0 | Train Loss: 8.3048 | Train Loss Unweighted: 2.9012 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6128 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1219 | Regression_eta: 0.0111 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6837 | Classification_binary: 2.6128 | Regression_cos_phi: 0.5082 | Regression_energy: 1.6417 | Regression_eta: 0.4657 | Regression_pt: 1.7161 | Regression_sin_phi: 0.6767 | LR: 6.06e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.3291s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1045 | max_forward: 0.0162 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3169 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4364.9 | uss_MB: 3886.5 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2838.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:21:32,234] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:32,290] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 12:21:32,371] INFO : Running validation on rank0 +rank0 [2026-10-28 12:21:32,371] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:21:32,371] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:21:33,453] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 12:21:33,598] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:21:33,731] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:21:33,807] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:21:33,876] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:21:33,944] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:21:34,014] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:21:34,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:21:34,197] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:21:34,321] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:21:34,443] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:21:34,516] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:21:34,587] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:21:34,656] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:21:34,725] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:21:34,794] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:21:34,906] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:21:34,913] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:21:34,916] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:21:34,919] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:21:34,920] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:21:34,923] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:21:34,926] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:21:34,927] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:21:34,929] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:21:34,932] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:21:34,933] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:21:34,936] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:21:34,937] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:21:34,940] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:21:34,942] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:21:34,945] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:21:40,985] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.3048 | Valid Loss=9.0555 | Train Loss Unweighted=2.9012 | Valid Loss Unweighted=3.0136 | Stale=0 | ETA=81.1m +rank0 [2026-10-28 12:21:40,995] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:41,057] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:21:41,061] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:41,222] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:42,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:43,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:43,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:44,245] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:44,377] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:44,380] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:44,435] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:21:44,438] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:44,535] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:45,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:46,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:46,995] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:47,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:47,748] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:47,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:21:47,806] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:21:47,808] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:21:47,999] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:21:49,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:21:50,020] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:21:50,532] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:21:51,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:21:51,167] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:21:51,176] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:51,429] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:21:51,893] INFO : Plotted number of elements +rank0 [2026-10-28 12:21:53,341] INFO : Plotted jets +rank0 [2026-10-28 12:21:54,437] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:54,471] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:54,696] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:21:55,075] INFO : Plotted number of elements +rank0 [2026-10-28 12:21:56,972] INFO : Plotted jets +rank0 [2026-10-28 12:21:58,086] INFO : Plotted jet ratio +rank0 [2026-10-28 12:21:58,119] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 12:21:58,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 12:21:58,739] INFO : Plotted number of elements +rank0 [2026-10-28 12:22:00,144] INFO : Plotted jets +rank0 [2026-10-28 12:22:01,243] INFO : Plotted jet ratio +rank0 [2026-10-28 12:22:12,647] INFO : Step 23100/50000 rank0 | Train Loss: 9.2078 | Train Loss Unweighted: 3.0558 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7110 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1469 | Regression_eta: 0.0095 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7018 | Classification_binary: 2.7110 | Regression_cos_phi: 0.4685 | Regression_energy: 1.9779 | Regression_eta: 0.3979 | Regression_pt: 2.1854 | Regression_sin_phi: 0.7653 | LR: 6.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4161s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1206 | max_loss: 0.0032 | max_backward: 0.0335 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3213.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:12,862] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:23,646] INFO : Step 23200/50000 rank0 | Train Loss: 8.6996 | Train Loss Unweighted: 2.9659 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6479 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1353 | Regression_eta: 0.0104 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7499 | Classification_binary: 2.6479 | Regression_cos_phi: 0.4622 | Regression_energy: 1.8215 | Regression_eta: 0.4368 | Regression_pt: 1.9575 | Regression_sin_phi: 0.6238 | LR: 6.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3255s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0324 | max_backward: 0.0335 | max_optimizer_step: 0.1169 | max_tensorboard_logging: 0.3913 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4565.3 | uss_MB: 4088.9 | child_rss_MB: 3213.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:23,858] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:34,858] INFO : Step 23300/50000 rank0 | Train Loss: 8.6422 | Train Loss Unweighted: 2.9491 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6257 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1395 | Regression_eta: 0.0091 | Regression_pt: 0.1388 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6754 | Classification_binary: 2.6257 | Regression_cos_phi: 0.5012 | Regression_energy: 1.8774 | Regression_eta: 0.3794 | Regression_pt: 2.0295 | Regression_sin_phi: 0.5535 | LR: 5.98e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4031s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1069 | max_loss: 0.0031 | max_backward: 0.0339 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3211.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:22:35,071] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:45,935] INFO : Step 23400/50000 rank0 | Train Loss: 8.7262 | Train Loss Unweighted: 2.8483 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5290 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1346 | Regression_eta: 0.0107 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7434 | Classification_binary: 2.5290 | Regression_cos_phi: 0.4939 | Regression_energy: 1.8114 | Regression_eta: 0.4504 | Regression_pt: 1.9552 | Regression_sin_phi: 0.7430 | LR: 5.95e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3905s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1197 | max_loss: 0.0031 | max_backward: 0.0334 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4565.3 | uss_MB: 4089.0 | child_rss_MB: 3211.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:46,148] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:56,954] INFO : Step 23500/50000 rank0 | Train Loss: 8.6694 | Train Loss Unweighted: 2.8873 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5754 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1314 | Regression_eta: 0.0134 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6989 | Classification_binary: 2.5754 | Regression_cos_phi: 0.4789 | Regression_energy: 1.7695 | Regression_eta: 0.5617 | Regression_pt: 1.8840 | Regression_sin_phi: 0.7009 | LR: 5.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3956s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.1005 | max_backward: 0.0336 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4565.3 | uss_MB: 4087.9 | child_rss_MB: 3206.4 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:22:57,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:08,066] INFO : Step 23600/50000 rank0 | Train Loss: 9.3664 | Train Loss Unweighted: 3.2217 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.8772 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1477 | Regression_eta: 0.0086 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7312 | Classification_binary: 2.8772 | Regression_cos_phi: 0.5057 | Regression_energy: 1.9878 | Regression_eta: 0.3612 | Regression_pt: 2.1656 | Regression_sin_phi: 0.7378 | LR: 5.90e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3435s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0671 | max_backward: 0.0337 | max_optimizer_step: 0.1173 | max_tensorboard_logging: 0.3781 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4438.2 | uss_MB: 3961.3 | child_rss_MB: 3205.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:08,280] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:19,304] INFO : Step 23700/50000 rank0 | Train Loss: 8.4570 | Train Loss Unweighted: 2.9799 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6745 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1311 | Regression_eta: 0.0088 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7596 | Classification_binary: 2.6745 | Regression_cos_phi: 0.4234 | Regression_energy: 1.7654 | Regression_eta: 0.3706 | Regression_pt: 1.8684 | Regression_sin_phi: 0.5950 | LR: 5.87e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3945s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1182 | max_loss: 0.0031 | max_backward: 0.0376 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3723 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4438.2 | uss_MB: 3965.0 | child_rss_MB: 3207.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:23:19,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:30,304] INFO : Step 23800/50000 rank0 | Train Loss: 9.0452 | Train Loss Unweighted: 3.0326 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7007 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1417 | Regression_eta: 0.0097 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7380 | Classification_binary: 2.7007 | Regression_cos_phi: 0.4831 | Regression_energy: 1.9071 | Regression_eta: 0.4053 | Regression_pt: 2.0531 | Regression_sin_phi: 0.7579 | LR: 5.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0159 | max_loss: 0.0453 | max_backward: 0.0326 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4439.2 | uss_MB: 3962.0 | child_rss_MB: 3207.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:23:30,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:41,187] INFO : Step 23900/50000 rank0 | Train Loss: 9.0834 | Train Loss Unweighted: 3.0904 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7674 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1349 | Regression_eta: 0.0114 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7750 | Classification_binary: 2.7674 | Regression_cos_phi: 0.5516 | Regression_energy: 1.8158 | Regression_eta: 0.4785 | Regression_pt: 1.9690 | Regression_sin_phi: 0.7261 | LR: 5.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2007s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0029 | max_backward: 0.0767 | max_optimizer_step: 0.1147 | max_tensorboard_logging: 0.2678 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4439.2 | uss_MB: 3963.2 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:23:41,402] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:52,324] INFO : Step 24000/50000 rank0 | Train Loss: 9.8334 | Train Loss Unweighted: 3.0472 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6680 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1626 | Regression_eta: 0.0113 | Regression_pt: 0.1622 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.6680 | Regression_cos_phi: 0.6280 | Regression_energy: 2.1890 | Regression_eta: 0.4720 | Regression_pt: 2.3717 | Regression_sin_phi: 0.7547 | LR: 5.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2600s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0032 | max_backward: 0.1086 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4439.2 | uss_MB: 3963.2 | child_rss_MB: 3232.0 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:23:52,538] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:52,557] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 12:23:52,638] INFO : Running validation on rank0 +rank0 [2026-10-28 12:23:52,638] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:23:52,638] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:23:53,822] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 12:23:53,967] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:23:54,099] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:23:54,174] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:23:54,243] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:23:54,310] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:23:54,379] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:23:54,449] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:23:54,561] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:23:54,684] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:23:54,807] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:23:54,878] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:23:54,947] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:23:55,015] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:23:55,085] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:23:55,155] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:23:55,270] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:23:55,277] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:23:55,279] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:23:55,282] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:23:55,283] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:23:55,286] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:23:55,289] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:23:55,290] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:23:55,292] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:23:55,295] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:23:55,296] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:23:55,298] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:23:55,300] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:23:55,302] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:23:55,305] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:23:55,308] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:24:01,241] INFO : VALIDATION | Step=24000/50000 | Train Loss=9.8334 | Valid Loss=9.0501 | Train Loss Unweighted=3.0472 | Valid Loss Unweighted=3.0260 | Stale=0 | ETA=77.4m +rank0 [2026-10-28 12:24:01,249] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:01,311] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:24:01,314] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:01,477] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:03,004] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:03,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:04,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:04,698] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:04,828] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:24:04,832] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:04,887] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:24:04,890] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:04,993] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:06,444] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:06,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:07,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:08,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:08,253] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:24:08,256] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:08,311] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:24:08,314] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:24:08,482] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:09,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:10,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:10,989] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:11,492] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:11,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:24:11,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:11,888] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:12,349] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:13,802] INFO : Plotted jets +rank0 [2026-10-28 12:24:14,902] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:14,935] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:15,160] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:15,539] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:17,435] INFO : Plotted jets +rank0 [2026-10-28 12:24:18,501] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:18,533] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 12:24:18,736] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 12:24:19,154] INFO : Plotted number of elements +rank0 [2026-10-28 12:24:20,557] INFO : Plotted jets +rank0 [2026-10-28 12:24:21,657] INFO : Plotted jet ratio +rank0 [2026-10-28 12:24:33,033] INFO : Step 24100/50000 rank0 | Train Loss: 9.2924 | Train Loss Unweighted: 3.0318 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6919 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1442 | Regression_eta: 0.0119 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7849 | Classification_binary: 2.6919 | Regression_cos_phi: 0.5482 | Regression_energy: 1.9410 | Regression_eta: 0.5004 | Regression_pt: 2.0609 | Regression_sin_phi: 0.7651 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1048 | max_forward: 0.0146 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2971 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3206.5 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:24:33,242] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:44,123] INFO : Step 24200/50000 rank0 | Train Loss: 8.6057 | Train Loss Unweighted: 2.9468 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6434 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1278 | Regression_eta: 0.0118 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7199 | Classification_binary: 2.6434 | Regression_cos_phi: 0.4161 | Regression_energy: 1.7201 | Regression_eta: 0.4940 | Regression_pt: 1.8298 | Regression_sin_phi: 0.7825 | LR: 5.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3208s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0401 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1211 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:24:44,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:55,162] INFO : Step 24300/50000 rank0 | Train Loss: 9.6636 | Train Loss Unweighted: 3.1530 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.7882 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1577 | Regression_eta: 0.0117 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7853 | Classification_binary: 2.7882 | Regression_cos_phi: 0.5293 | Regression_energy: 2.1233 | Regression_eta: 0.4887 | Regression_pt: 2.2481 | Regression_sin_phi: 0.7008 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3168s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0161 | max_backward: 0.0337 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3209.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:24:55,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:06,303] INFO : Step 24400/50000 rank0 | Train Loss: 8.5365 | Train Loss Unweighted: 3.0744 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.7745 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1262 | Regression_eta: 0.0106 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6516 | Classification_binary: 2.7745 | Regression_cos_phi: 0.4420 | Regression_energy: 1.6992 | Regression_eta: 0.4445 | Regression_pt: 1.8624 | Regression_sin_phi: 0.6623 | LR: 5.67e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3645s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0381 | max_backward: 0.0326 | max_optimizer_step: 0.1208 | max_tensorboard_logging: 0.3921 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3208.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:06,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:17,198] INFO : Step 24500/50000 rank0 | Train Loss: 8.5981 | Train Loss Unweighted: 2.8085 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.4983 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1298 | Regression_eta: 0.0107 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7910 | Classification_binary: 2.4983 | Regression_cos_phi: 0.6159 | Regression_energy: 1.7469 | Regression_eta: 0.4466 | Regression_pt: 1.8577 | Regression_sin_phi: 0.6417 | LR: 5.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.2482s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0031 | max_backward: 0.0746 | max_optimizer_step: 0.1184 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4628.3 | uss_MB: 4151.4 | child_rss_MB: 3224.5 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:25:17,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:28,208] INFO : Step 24600/50000 rank0 | Train Loss: 9.3902 | Train Loss Unweighted: 3.0343 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6847 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1490 | Regression_eta: 0.0117 | Regression_pt: 0.1470 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7400 | Classification_binary: 2.6847 | Regression_cos_phi: 0.6020 | Regression_energy: 2.0056 | Regression_eta: 0.4907 | Regression_pt: 2.1504 | Regression_sin_phi: 0.7168 | LR: 5.61e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.2991s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.2216 | max_tensorboard_logging: 0.2926 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4513.8 | uss_MB: 4036.5 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:25:28,433] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:39,332] INFO : Step 24700/50000 rank0 | Train Loss: 9.0457 | Train Loss Unweighted: 3.0486 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7218 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1373 | Regression_eta: 0.0092 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7666 | Classification_binary: 2.7218 | Regression_cos_phi: 0.5160 | Regression_energy: 1.8489 | Regression_eta: 0.3838 | Regression_pt: 2.0234 | Regression_sin_phi: 0.7853 | LR: 5.58e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3056s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4513.8 | uss_MB: 4036.6 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:25:39,559] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:25:50,205] INFO : Step 24800/50000 rank0 | Train Loss: 8.7281 | Train Loss Unweighted: 2.9290 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6053 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1372 | Regression_eta: 0.0110 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6748 | Classification_binary: 2.6053 | Regression_cos_phi: 0.5224 | Regression_energy: 1.8467 | Regression_eta: 0.4595 | Regression_pt: 2.0281 | Regression_sin_phi: 0.5912 | LR: 5.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2510s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0247 | max_forward: 0.0154 | max_loss: 0.0032 | max_backward: 0.0341 | max_optimizer_step: 0.1200 | max_tensorboard_logging: 0.3101 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4513.8 | uss_MB: 4036.3 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:25:50,420] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:01,290] INFO : Step 24900/50000 rank0 | Train Loss: 8.4344 | Train Loss Unweighted: 2.9049 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6030 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1267 | Regression_eta: 0.0095 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7053 | Classification_binary: 2.6030 | Regression_cos_phi: 0.4542 | Regression_energy: 1.7052 | Regression_eta: 0.4000 | Regression_pt: 1.8677 | Regression_sin_phi: 0.6990 | LR: 5.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1016 | max_loss: 0.0032 | max_backward: 0.0333 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4513.8 | uss_MB: 4036.3 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:26:01,504] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:12,274] INFO : Step 25000/50000 rank0 | Train Loss: 8.5499 | Train Loss Unweighted: 2.9449 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6413 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1283 | Regression_eta: 0.0102 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6905 | Classification_binary: 2.6413 | Regression_cos_phi: 0.5190 | Regression_energy: 1.7269 | Regression_eta: 0.4286 | Regression_pt: 1.8491 | Regression_sin_phi: 0.6945 | LR: 5.50e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3526s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0133 | max_loss: 0.0714 | max_backward: 0.0327 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.3685 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4513.8 | uss_MB: 4036.6 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2568.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:26:12,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:12,505] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 12:26:12,690] INFO : Running validation on rank0 +rank0 [2026-10-28 12:26:12,690] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:26:12,690] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:26:13,770] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 12:26:13,914] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:26:14,046] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:26:14,119] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:26:14,188] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:26:14,255] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:26:14,325] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:26:14,393] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:26:14,505] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:26:14,629] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:26:14,750] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:26:14,822] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:26:14,892] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:26:14,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:26:15,029] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:26:15,098] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:26:15,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:26:15,209] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:26:15,211] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:26:15,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:26:15,215] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:26:15,217] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:26:15,220] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:26:15,221] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:26:15,223] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:26:15,225] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:26:15,227] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:26:15,230] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:26:15,232] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:26:15,235] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:26:15,236] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:26:15,238] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:26:21,015] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.5499 | Valid Loss=8.9006 | Train Loss Unweighted=2.9449 | Valid Loss Unweighted=2.9816 | Stale=0 | ETA=73.8m +rank0 [2026-10-28 12:26:21,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:21,085] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:26:21,089] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:21,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:22,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:23,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:23,774] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:24,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:24,522] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:24,525] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:24,581] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:26:24,584] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:24,752] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:26,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:26,805] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:27,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:27,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:28,090] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:28,093] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:26:28,149] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:26:28,153] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:26:28,250] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:26:29,581] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:26:30,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:26:30,689] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:26:31,197] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:26:31,333] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:26:31,340] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:31,591] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:26:32,047] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:33,488] INFO : Plotted jets +rank0 [2026-10-28 12:26:34,594] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:34,627] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:34,857] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:26:35,233] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:37,116] INFO : Plotted jets +rank0 [2026-10-28 12:26:38,175] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:38,208] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 12:26:38,408] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 12:26:38,822] INFO : Plotted number of elements +rank0 [2026-10-28 12:26:40,219] INFO : Plotted jets +rank0 [2026-10-28 12:26:41,312] INFO : Plotted jet ratio +rank0 [2026-10-28 12:26:52,539] INFO : Step 25100/50000 rank0 | Train Loss: 9.3452 | Train Loss Unweighted: 3.0107 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1528 | Regression_eta: 0.0102 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7839 | Classification_binary: 2.6540 | Regression_cos_phi: 0.4921 | Regression_energy: 2.0576 | Regression_eta: 0.4280 | Regression_pt: 2.2339 | Regression_sin_phi: 0.6957 | LR: 5.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0602 | max_backward: 0.0337 | max_optimizer_step: 0.1226 | max_tensorboard_logging: 0.3646 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3199.9 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:26:52,753] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:03,673] INFO : Step 25200/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.9470 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6397 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1300 | Regression_eta: 0.0102 | Regression_pt: 0.1285 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7365 | Classification_binary: 2.6397 | Regression_cos_phi: 0.4627 | Regression_energy: 1.7502 | Regression_eta: 0.4257 | Regression_pt: 1.8794 | Regression_sin_phi: 0.6661 | LR: 5.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3505s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0662 | max_backward: 0.0341 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.3674 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3199.1 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:27:03,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:14,605] INFO : Step 25300/50000 rank0 | Train Loss: 8.5483 | Train Loss Unweighted: 2.9748 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6578 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1361 | Regression_eta: 0.0085 | Regression_pt: 0.1357 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7473 | Classification_binary: 2.6578 | Regression_cos_phi: 0.4192 | Regression_energy: 1.8317 | Regression_eta: 0.3581 | Regression_pt: 1.9841 | Regression_sin_phi: 0.5501 | LR: 5.42e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0137 | max_loss: 0.0032 | max_backward: 0.0704 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4714.0 | uss_MB: 4233.7 | child_rss_MB: 3223.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:27:14,818] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:25,640] INFO : Step 25400/50000 rank0 | Train Loss: 9.0436 | Train Loss Unweighted: 3.0074 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6784 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1386 | Regression_eta: 0.0100 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7470 | Classification_binary: 2.6784 | Regression_cos_phi: 0.5386 | Regression_energy: 1.8661 | Regression_eta: 0.4172 | Regression_pt: 2.0311 | Regression_sin_phi: 0.7652 | LR: 5.39e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.1986 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4714.0 | uss_MB: 4233.9 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:25,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:36,756] INFO : Step 25500/50000 rank0 | Train Loss: 8.6778 | Train Loss Unweighted: 2.9564 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6444 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1314 | Regression_eta: 0.0104 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.6444 | Regression_cos_phi: 0.4234 | Regression_energy: 1.7690 | Regression_eta: 0.4364 | Regression_pt: 1.9156 | Regression_sin_phi: 0.7391 | LR: 5.36e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3051s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0168 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4714.0 | uss_MB: 4233.9 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:36,984] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:47,713] INFO : Step 25600/50000 rank0 | Train Loss: 8.6983 | Train Loss Unweighted: 2.9770 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6611 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1340 | Regression_eta: 0.0110 | Regression_pt: 0.1338 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6931 | Classification_binary: 2.6611 | Regression_cos_phi: 0.4315 | Regression_energy: 1.8039 | Regression_eta: 0.4614 | Regression_pt: 1.9566 | Regression_sin_phi: 0.6908 | LR: 5.33e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.1877s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2761 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4593.8 | uss_MB: 4112.3 | child_rss_MB: 3203.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:47,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:58,726] INFO : Step 25700/50000 rank0 | Train Loss: 9.0989 | Train Loss Unweighted: 3.0015 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6423 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1563 | Regression_eta: 0.0087 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7876 | Classification_binary: 2.6423 | Regression_cos_phi: 0.3569 | Regression_energy: 2.1037 | Regression_eta: 0.3628 | Regression_pt: 2.3054 | Regression_sin_phi: 0.5401 | LR: 5.30e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3663s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0802 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1192 | max_tensorboard_logging: 0.3744 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3204.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:27:58,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:09,851] INFO : Step 25800/50000 rank0 | Train Loss: 8.9372 | Train Loss Unweighted: 2.8968 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.5623 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1428 | Regression_eta: 0.0104 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7489 | Classification_binary: 2.5623 | Regression_cos_phi: 0.5540 | Regression_energy: 1.9230 | Regression_eta: 0.4365 | Regression_pt: 2.0577 | Regression_sin_phi: 0.6547 | LR: 5.27e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3830s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0960 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1205 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:28:10,064] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:20,663] INFO : Step 25900/50000 rank0 | Train Loss: 9.5949 | Train Loss Unweighted: 3.1370 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7832 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1512 | Regression_eta: 0.0112 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7232 | Classification_binary: 2.7832 | Regression_cos_phi: 0.5369 | Regression_energy: 2.0358 | Regression_eta: 0.4698 | Regression_pt: 2.1842 | Regression_sin_phi: 0.8618 | LR: 5.25e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2329s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0658 | max_optimizer_step: 0.1197 | max_tensorboard_logging: 0.2762 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.4 | child_rss_MB: 3224.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:28:20,876] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:31,696] INFO : Step 26000/50000 rank0 | Train Loss: 9.4804 | Train Loss Unweighted: 3.0324 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6751 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1532 | Regression_eta: 0.0113 | Regression_pt: 0.1502 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7788 | Classification_binary: 2.6751 | Regression_cos_phi: 0.5717 | Regression_energy: 2.0618 | Regression_eta: 0.4742 | Regression_pt: 2.1967 | Regression_sin_phi: 0.7220 | LR: 5.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2703s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0031 | max_backward: 0.0338 | max_optimizer_step: 0.2014 | max_tensorboard_logging: 0.2709 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4593.8 | uss_MB: 4112.5 | child_rss_MB: 3203.9 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:28:31,925] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:31,941] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 12:28:32,021] INFO : Running validation on rank0 +rank0 [2026-10-28 12:28:32,021] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:28:32,021] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:28:33,202] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 12:28:33,352] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:28:33,484] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:28:33,558] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:28:33,627] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:28:33,695] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:28:33,765] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:28:33,834] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:28:33,948] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:28:34,073] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:28:34,196] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:28:34,268] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:28:34,338] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:28:34,406] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:28:34,475] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:28:34,543] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:28:34,653] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:28:34,659] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:28:34,662] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:28:34,665] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:28:34,668] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:28:34,669] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:28:34,672] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:28:34,672] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:28:34,675] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:28:34,678] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:28:34,679] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:28:34,682] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:28:34,684] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:28:34,685] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:28:34,688] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:28:34,691] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:28:40,639] INFO : VALIDATION | Step=26000/50000 | Train Loss=9.4804 | Valid Loss=8.8191 | Train Loss Unweighted=3.0324 | Valid Loss Unweighted=2.9569 | Stale=0 | ETA=70.3m +rank0 [2026-10-28 12:28:40,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:40,711] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:28:40,714] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:40,784] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:42,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:42,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:43,507] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:44,113] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:44,276] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:28:44,279] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:44,334] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:28:44,337] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:44,455] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:46,183] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:46,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:47,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:47,861] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:47,976] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:28:47,979] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:28:48,034] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:28:48,037] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:28:48,140] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:28:49,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:28:50,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:28:50,745] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:28:51,330] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:28:51,466] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:28:51,473] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:51,731] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:28:52,189] INFO : Plotted number of elements +rank0 [2026-10-28 12:28:53,634] INFO : Plotted jets +rank0 [2026-10-28 12:28:54,735] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:54,767] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:54,992] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:28:55,375] INFO : Plotted number of elements +rank0 [2026-10-28 12:28:57,255] INFO : Plotted jets +rank0 [2026-10-28 12:28:58,373] INFO : Plotted jet ratio +rank0 [2026-10-28 12:28:58,404] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 12:28:58,608] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 12:28:59,024] INFO : Plotted number of elements +rank0 [2026-10-28 12:29:00,425] INFO : Plotted jets +rank0 [2026-10-28 12:29:01,530] INFO : Plotted jet ratio +rank0 [2026-10-28 12:29:12,920] INFO : Step 26100/50000 rank0 | Train Loss: 9.3825 | Train Loss Unweighted: 2.9760 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6205 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1528 | Regression_eta: 0.0091 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7711 | Classification_binary: 2.6205 | Regression_cos_phi: 0.6339 | Regression_energy: 2.0566 | Regression_eta: 0.3833 | Regression_pt: 2.1951 | Regression_sin_phi: 0.7221 | LR: 5.19e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1103 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1221 | max_tensorboard_logging: 0.3668 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4789.6 | uss_MB: 4309.9 | child_rss_MB: 3204.0 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:29:13,134] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:24,100] INFO : Step 26200/50000 rank0 | Train Loss: 9.0844 | Train Loss Unweighted: 2.9533 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6276 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1364 | Regression_eta: 0.0123 | Regression_pt: 0.1351 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6725 | Classification_binary: 2.6276 | Regression_cos_phi: 0.6333 | Regression_energy: 1.8368 | Regression_eta: 0.5151 | Regression_pt: 1.9752 | Regression_sin_phi: 0.8240 | LR: 5.16e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0516 | max_loss: 0.0032 | max_backward: 0.0341 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3652 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4789.6 | uss_MB: 4309.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:29:24,315] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:35,273] INFO : Step 26300/50000 rank0 | Train Loss: 8.3810 | Train Loss Unweighted: 2.8987 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6005 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1239 | Regression_eta: 0.0098 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7266 | Classification_binary: 2.6005 | Regression_cos_phi: 0.5110 | Regression_energy: 1.6684 | Regression_eta: 0.4100 | Regression_pt: 1.8395 | Regression_sin_phi: 0.6252 | LR: 5.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0596 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 4789.6 | uss_MB: 4310.0 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:29:35,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:46,315] INFO : Step 26400/50000 rank0 | Train Loss: 8.8001 | Train Loss Unweighted: 2.9750 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6523 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1383 | Regression_eta: 0.0094 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7000 | Classification_binary: 2.6523 | Regression_cos_phi: 0.4713 | Regression_energy: 1.8621 | Regression_eta: 0.3940 | Regression_pt: 1.9974 | Regression_sin_phi: 0.7230 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0171 | max_loss: 0.0475 | max_backward: 0.0385 | max_optimizer_step: 0.1223 | max_tensorboard_logging: 0.3656 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4789.6 | uss_MB: 4312.0 | child_rss_MB: 3201.4 | allocated_MB: 210.7 | reserved_MB: 2748.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:29:46,532] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:57,864] INFO : Step 26500/50000 rank0 | Train Loss: 8.7020 | Train Loss Unweighted: 2.8341 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5011 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1444 | Regression_eta: 0.0100 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6866 | Classification_binary: 2.5011 | Regression_cos_phi: 0.4269 | Regression_energy: 1.9444 | Regression_eta: 0.4199 | Regression_pt: 2.0813 | Regression_sin_phi: 0.6418 | LR: 5.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3550s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.2302 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 4789.6 | uss_MB: 4310.0 | child_rss_MB: 3200.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:29:58,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:09,041] INFO : Step 26600/50000 rank0 | Train Loss: 9.3045 | Train Loss Unweighted: 3.0020 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6591 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1454 | Regression_eta: 0.0136 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7117 | Classification_binary: 2.6591 | Regression_cos_phi: 0.5406 | Regression_energy: 1.9581 | Regression_eta: 0.5701 | Regression_pt: 2.0930 | Regression_sin_phi: 0.7720 | LR: 5.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3492s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.2076 | max_tensorboard_logging: 0.3068 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4664.1 | uss_MB: 4183.5 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:30:09,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:20,302] INFO : Step 26700/50000 rank0 | Train Loss: 8.9748 | Train Loss Unweighted: 2.9701 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6266 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1486 | Regression_eta: 0.0095 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6891 | Classification_binary: 2.6266 | Regression_cos_phi: 0.4680 | Regression_energy: 2.0010 | Regression_eta: 0.3965 | Regression_pt: 2.1744 | Regression_sin_phi: 0.6191 | LR: 5.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0134 | max_loss: 0.0030 | max_backward: 0.1399 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.2918 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4664.1 | uss_MB: 4184.5 | child_rss_MB: 3225.2 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:20,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:31,365] INFO : Step 26800/50000 rank0 | Train Loss: 9.3820 | Train Loss Unweighted: 3.0534 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.7114 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1429 | Regression_eta: 0.0112 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.8169 | Classification_binary: 2.7114 | Regression_cos_phi: 0.5681 | Regression_energy: 1.9235 | Regression_eta: 0.4694 | Regression_pt: 2.0965 | Regression_sin_phi: 0.7961 | LR: 4.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3389s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0031 | max_backward: 0.0325 | max_optimizer_step: 0.2187 | max_tensorboard_logging: 0.2885 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4664.1 | uss_MB: 4184.9 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:31,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:42,516] INFO : Step 26900/50000 rank0 | Train Loss: 9.4787 | Train Loss Unweighted: 3.0190 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6638 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1520 | Regression_eta: 0.0129 | Regression_pt: 0.1485 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7326 | Classification_binary: 2.6638 | Regression_cos_phi: 0.6039 | Regression_energy: 2.0463 | Regression_eta: 0.5407 | Regression_pt: 2.1719 | Regression_sin_phi: 0.7194 | LR: 4.96e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.2396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0031 | max_backward: 0.0330 | max_optimizer_step: 0.1651 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4664.1 | uss_MB: 4185.0 | child_rss_MB: 3199.2 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:30:42,748] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:53,401] INFO : Step 27000/50000 rank0 | Train Loss: 9.7092 | Train Loss Unweighted: 3.1117 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.7499 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1539 | Regression_eta: 0.0149 | Regression_pt: 0.1504 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7897 | Classification_binary: 2.7499 | Regression_cos_phi: 0.5597 | Regression_energy: 2.0725 | Regression_eta: 0.6231 | Regression_pt: 2.2000 | Regression_sin_phi: 0.7143 | LR: 4.94e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3011s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0579 | max_forward: 0.0158 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1201 | max_tensorboard_logging: 0.3031 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4664.1 | uss_MB: 4185.0 | child_rss_MB: 3198.4 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:30:53,618] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:53,663] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 12:30:53,744] INFO : Running validation on rank0 +rank0 [2026-10-28 12:30:53,744] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:30:53,744] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:30:54,830] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 12:30:54,976] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:30:55,107] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:30:55,182] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:30:55,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:30:55,320] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:30:55,390] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:30:55,460] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:30:55,573] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:30:55,698] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:30:55,820] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:30:55,893] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:30:55,964] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:30:56,034] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:30:56,103] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:30:56,172] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:30:56,288] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:30:56,294] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:30:56,297] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:30:56,300] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:30:56,301] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:30:56,303] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:30:56,306] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:30:56,307] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:30:56,310] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:30:56,313] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:30:56,314] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:30:56,316] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:30:56,319] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:30:56,322] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:30:56,322] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:30:56,325] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:31:02,359] INFO : VALIDATION | Step=27000/50000 | Train Loss=9.7092 | Valid Loss=8.8099 | Train Loss Unweighted=3.1117 | Valid Loss Unweighted=2.9414 | Stale=0 | ETA=66.9m +rank0 [2026-10-28 12:31:02,368] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:02,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:31:02,442] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:02,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:04,012] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:04,582] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:05,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:05,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:05,827] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:05,831] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:05,888] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:31:05,891] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:05,985] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:07,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:08,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:08,596] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:09,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:09,272] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:09,275] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:09,329] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:31:09,332] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:31:09,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:31:10,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:31:11,500] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:31:12,011] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:31:12,514] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:31:12,670] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:31:12,677] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:12,927] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:13,385] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:14,828] INFO : Plotted jets +rank0 [2026-10-28 12:31:15,936] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:15,969] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:16,195] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:31:16,575] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:18,449] INFO : Plotted jets +rank0 [2026-10-28 12:31:19,571] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:19,605] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 12:31:19,806] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 12:31:20,222] INFO : Plotted number of elements +rank0 [2026-10-28 12:31:21,620] INFO : Plotted jets +rank0 [2026-10-28 12:31:22,725] INFO : Plotted jet ratio +rank0 [2026-10-28 12:31:33,905] INFO : Step 27100/50000 rank0 | Train Loss: 8.8459 | Train Loss Unweighted: 2.8971 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5642 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1422 | Regression_eta: 0.0093 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7237 | Classification_binary: 2.5642 | Regression_cos_phi: 0.4974 | Regression_energy: 1.9144 | Regression_eta: 0.3894 | Regression_pt: 2.0829 | Regression_sin_phi: 0.6740 | LR: 4.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4299s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1267 | max_loss: 0.0034 | max_backward: 0.0386 | max_optimizer_step: 0.1206 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4857.2 | uss_MB: 4381.4 | child_rss_MB: 3200.8 | allocated_MB: 210.7 | reserved_MB: 2768.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:31:34,118] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:45,031] INFO : Step 27200/50000 rank0 | Train Loss: 8.7159 | Train Loss Unweighted: 2.8628 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5394 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1378 | Regression_eta: 0.0091 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7149 | Classification_binary: 2.5394 | Regression_cos_phi: 0.4609 | Regression_energy: 1.8552 | Regression_eta: 0.3809 | Regression_pt: 2.0101 | Regression_sin_phi: 0.7546 | LR: 4.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0413 | max_loss: 0.0033 | max_backward: 0.0331 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3712 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4857.2 | uss_MB: 4379.3 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2768.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:31:45,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:56,166] INFO : Step 27300/50000 rank0 | Train Loss: 8.4225 | Train Loss Unweighted: 2.8975 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5916 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1309 | Regression_eta: 0.0098 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.5916 | Regression_cos_phi: 0.4176 | Regression_energy: 1.7628 | Regression_eta: 0.4127 | Regression_pt: 1.8820 | Regression_sin_phi: 0.6641 | LR: 4.85e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3797s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0512 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3201.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:31:56,380] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:07,283] INFO : Step 27400/50000 rank0 | Train Loss: 8.7941 | Train Loss Unweighted: 2.9234 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6138 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1281 | Regression_eta: 0.0115 | Regression_pt: 0.1269 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7778 | Classification_binary: 2.6138 | Regression_cos_phi: 0.5622 | Regression_energy: 1.7241 | Regression_eta: 0.4807 | Regression_pt: 1.8557 | Regression_sin_phi: 0.7798 | LR: 4.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2955s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0576 | max_forward: 0.0157 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1242 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3200.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:07,497] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:18,272] INFO : Step 27500/50000 rank0 | Train Loss: 8.4812 | Train Loss Unweighted: 2.8603 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5534 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1301 | Regression_eta: 0.0083 | Regression_pt: 0.1296 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7229 | Classification_binary: 2.5534 | Regression_cos_phi: 0.4173 | Regression_energy: 1.7509 | Regression_eta: 0.3472 | Regression_pt: 1.8953 | Regression_sin_phi: 0.7942 | LR: 4.80e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0406 | max_backward: 0.0333 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3762 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4857.2 | uss_MB: 4379.4 | child_rss_MB: 3200.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:18,484] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:29,396] INFO : Step 27600/50000 rank0 | Train Loss: 9.1038 | Train Loss Unweighted: 2.9487 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6253 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1341 | Regression_eta: 0.0141 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7621 | Classification_binary: 2.6253 | Regression_cos_phi: 0.6196 | Regression_energy: 1.8049 | Regression_eta: 0.5919 | Regression_pt: 1.9235 | Regression_sin_phi: 0.7764 | LR: 4.77e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3896s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0563 | max_backward: 0.0324 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3863 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 4735.5 | uss_MB: 4256.6 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:29,611] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:32:40,592] INFO : Step 27700/50000 rank0 | Train Loss: 8.3890 | Train Loss Unweighted: 2.8754 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5651 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1306 | Regression_eta: 0.0091 | Regression_pt: 0.1344 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7154 | Classification_binary: 2.5651 | Regression_cos_phi: 0.4244 | Regression_energy: 1.7588 | Regression_eta: 0.3796 | Regression_pt: 1.9651 | Regression_sin_phi: 0.5806 | LR: 4.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3685s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0664 | max_loss: 0.0030 | max_backward: 0.0339 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.3739 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4735.5 | uss_MB: 4257.5 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:32:40,803] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:32:51,591] INFO : Step 27800/50000 rank0 | Train Loss: 8.7628 | Train Loss Unweighted: 2.9021 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5668 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1454 | Regression_eta: 0.0090 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6817 | Classification_binary: 2.5668 | Regression_cos_phi: 0.3922 | Regression_energy: 1.9575 | Regression_eta: 0.3767 | Regression_pt: 2.1240 | Regression_sin_phi: 0.6638 | LR: 4.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0152 | max_loss: 0.0108 | max_backward: 0.0337 | max_optimizer_step: 0.1223 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4735.5 | uss_MB: 4257.6 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:32:51,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:02,810] INFO : Step 27900/50000 rank0 | Train Loss: 8.2349 | Train Loss Unweighted: 2.8685 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.5744 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1238 | Regression_eta: 0.0102 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6050 | Classification_binary: 2.5744 | Regression_cos_phi: 0.4691 | Regression_energy: 1.6668 | Regression_eta: 0.4283 | Regression_pt: 1.8336 | Regression_sin_phi: 0.6578 | LR: 4.68e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3417s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0431 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4735.5 | uss_MB: 4257.7 | child_rss_MB: 3202.0 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:33:03,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:13,919] INFO : Step 28000/50000 rank0 | Train Loss: 8.6612 | Train Loss Unweighted: 2.8514 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5285 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1360 | Regression_eta: 0.0109 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7079 | Classification_binary: 2.5285 | Regression_cos_phi: 0.4541 | Regression_energy: 1.8315 | Regression_eta: 0.4568 | Regression_pt: 2.0243 | Regression_sin_phi: 0.6581 | LR: 4.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1196 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1217 | max_tensorboard_logging: 0.3708 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4735.5 | uss_MB: 4257.7 | child_rss_MB: 3200.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:14,133] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:14,222] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 12:33:14,304] INFO : Running validation on rank0 +rank0 [2026-10-28 12:33:14,304] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:33:14,304] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:33:15,389] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 12:33:15,533] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:33:15,665] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:33:15,739] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:33:15,809] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:33:15,876] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:33:15,946] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:33:16,015] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:33:16,127] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:33:16,252] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:33:16,373] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:33:16,446] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:33:16,517] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:33:16,585] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:33:16,654] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:33:16,723] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:33:16,840] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:33:16,847] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:33:16,849] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:33:16,852] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:33:16,853] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:33:16,856] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:33:16,858] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:33:16,861] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:33:16,862] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:33:16,865] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:33:16,867] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:33:16,868] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:33:16,871] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:33:16,872] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:33:16,874] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:33:16,877] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:33:22,922] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.6612 | Valid Loss=8.7574 | Train Loss Unweighted=2.8514 | Valid Loss Unweighted=2.9339 | Stale=0 | ETA=63.5m +rank0 [2026-10-28 12:33:22,931] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:22,993] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:33:22,997] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:23,167] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:24,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:25,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:25,762] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:26,315] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:26,462] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:26,466] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:26,532] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:33:26,535] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:26,630] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:27,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:28,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:29,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:29,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:29,892] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:29,895] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:33:29,948] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:33:29,951] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:33:30,074] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:33:31,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:33:32,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:33:32,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:33:33,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:33:33,286] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:33:33,294] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:33,552] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:33:34,011] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:35,471] INFO : Plotted jets +rank0 [2026-10-28 12:33:36,575] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:36,606] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:36,831] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:33:37,212] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:39,120] INFO : Plotted jets +rank0 [2026-10-28 12:33:40,235] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:40,267] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 12:33:40,474] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 12:33:40,890] INFO : Plotted number of elements +rank0 [2026-10-28 12:33:42,297] INFO : Plotted jets +rank0 [2026-10-28 12:33:43,394] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:54,756] INFO : Step 28100/50000 rank0 | Train Loss: 8.2969 | Train Loss Unweighted: 2.8741 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5658 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1330 | Regression_eta: 0.0083 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7055 | Classification_binary: 2.5658 | Regression_cos_phi: 0.4156 | Regression_energy: 1.7900 | Regression_eta: 0.3489 | Regression_pt: 1.9259 | Regression_sin_phi: 0.5453 | LR: 4.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3363s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0153 | max_loss: 0.0145 | max_backward: 0.0336 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3677 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.2 | child_rss_MB: 3201.4 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:33:54,968] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:06,022] INFO : Step 28200/50000 rank0 | Train Loss: 8.8105 | Train Loss Unweighted: 3.0099 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.6870 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1383 | Regression_eta: 0.0093 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7779 | Classification_binary: 2.6870 | Regression_cos_phi: 0.4373 | Regression_energy: 1.8617 | Regression_eta: 0.3880 | Regression_pt: 1.9886 | Regression_sin_phi: 0.6702 | LR: 4.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3911s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0948 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3686 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4932.5 | uss_MB: 4456.3 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:06,235] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 3% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:34:17,181] INFO : Step 28300/50000 rank0 | Train Loss: 8.5038 | Train Loss Unweighted: 2.9983 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6963 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1292 | Regression_eta: 0.0092 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7273 | Classification_binary: 2.6963 | Regression_cos_phi: 0.4753 | Regression_energy: 1.7395 | Regression_eta: 0.3876 | Regression_pt: 1.8300 | Regression_sin_phi: 0.6479 | LR: 4.57e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0922 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3203.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:34:17,396] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:28,324] INFO : Step 28400/50000 rank0 | Train Loss: 9.0222 | Train Loss Unweighted: 2.9260 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.5867 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1462 | Regression_eta: 0.0113 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7713 | Classification_binary: 2.5867 | Regression_cos_phi: 0.4744 | Regression_energy: 1.9679 | Regression_eta: 0.4717 | Regression_pt: 2.0738 | Regression_sin_phi: 0.6762 | LR: 4.55e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1227 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3643 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3204.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:34:28,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:39,463] INFO : Step 28500/50000 rank0 | Train Loss: 8.7565 | Train Loss Unweighted: 2.9061 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5880 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1350 | Regression_eta: 0.0090 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6799 | Classification_binary: 2.5880 | Regression_cos_phi: 0.5340 | Regression_energy: 1.8174 | Regression_eta: 0.3780 | Regression_pt: 1.9618 | Regression_sin_phi: 0.7973 | LR: 4.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1218 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3700 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:34:39,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:50,634] INFO : Step 28600/50000 rank0 | Train Loss: 8.5012 | Train Loss Unweighted: 2.8955 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5916 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1298 | Regression_eta: 0.0104 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6453 | Classification_binary: 2.5916 | Regression_cos_phi: 0.5026 | Regression_energy: 1.7476 | Regression_eta: 0.4353 | Regression_pt: 1.8458 | Regression_sin_phi: 0.7331 | LR: 4.49e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3527s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0236 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4932.5 | uss_MB: 4456.4 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:34:50,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:01,646] INFO : Step 28700/50000 rank0 | Train Loss: 9.1998 | Train Loss Unweighted: 2.9615 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.6217 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1452 | Regression_eta: 0.0119 | Regression_pt: 0.1413 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7488 | Classification_binary: 2.6217 | Regression_cos_phi: 0.5026 | Regression_energy: 1.9545 | Regression_eta: 0.5006 | Regression_pt: 2.0662 | Regression_sin_phi: 0.8053 | LR: 4.46e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3616s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0439 | max_backward: 0.0338 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3752 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4808.5 | uss_MB: 4332.5 | child_rss_MB: 3201.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:01,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:12,602] INFO : Step 28800/50000 rank0 | Train Loss: 8.3481 | Train Loss Unweighted: 2.7553 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4499 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1284 | Regression_eta: 0.0101 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6734 | Classification_binary: 2.4499 | Regression_cos_phi: 0.4899 | Regression_energy: 1.7291 | Regression_eta: 0.4240 | Regression_pt: 1.8900 | Regression_sin_phi: 0.6917 | LR: 4.44e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2370s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0430 | max_optimizer_step: 0.1237 | max_tensorboard_logging: 0.2794 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4808.5 | uss_MB: 4332.5 | child_rss_MB: 3222.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:35:12,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:23,640] INFO : Step 28900/50000 rank0 | Train Loss: 8.4294 | Train Loss Unweighted: 2.8640 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5543 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1328 | Regression_eta: 0.0101 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.5543 | Regression_cos_phi: 0.4486 | Regression_energy: 1.7873 | Regression_eta: 0.4229 | Regression_pt: 1.9168 | Regression_sin_phi: 0.6373 | LR: 4.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0160 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1585 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4808.5 | uss_MB: 4332.6 | child_rss_MB: 3200.3 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:35:23,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:34,599] INFO : Step 29000/50000 rank0 | Train Loss: 8.5618 | Train Loss Unweighted: 2.8650 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5619 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1266 | Regression_eta: 0.0107 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.5619 | Regression_cos_phi: 0.4800 | Regression_energy: 1.7046 | Regression_eta: 0.4503 | Regression_pt: 1.8342 | Regression_sin_phi: 0.8087 | LR: 4.38e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.1975s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0156 | max_loss: 0.0032 | max_backward: 0.0324 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.2726 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4808.5 | uss_MB: 4332.6 | child_rss_MB: 3205.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:35:34,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:34,832] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 12:35:34,915] INFO : Running validation on rank0 +rank0 [2026-10-28 12:35:34,915] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:35:34,915] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:35:36,003] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 12:35:36,148] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:35:36,280] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:35:36,354] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:35:36,422] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:35:36,490] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:35:36,559] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:35:36,629] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:35:36,741] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:35:36,866] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:35:36,988] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:35:37,060] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:35:37,130] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:35:37,197] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:35:37,266] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:35:37,335] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:35:37,452] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:35:37,459] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:35:37,462] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:35:37,465] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:35:37,468] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:35:37,470] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:35:37,471] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:35:37,474] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:35:37,477] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:35:37,479] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:35:37,480] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:35:37,483] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:35:37,486] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:35:37,488] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:35:37,491] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:35:37,492] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:35:43,296] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.5618 | Valid Loss=8.7153 | Train Loss Unweighted=2.8650 | Valid Loss Unweighted=2.9113 | Stale=0 | ETA=60.2m +rank0 [2026-10-28 12:35:43,305] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:43,392] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:35:43,396] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:43,566] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:44,927] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:45,496] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:46,074] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:46,672] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:46,825] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:35:46,828] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:46,885] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:35:46,887] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:46,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:48,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:49,098] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:49,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:50,252] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:50,406] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:35:50,410] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:35:50,463] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:35:50,467] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:35:50,581] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:35:51,946] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:35:52,456] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:35:52,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:35:53,545] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:35:53,718] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:35:53,726] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:53,983] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:35:54,436] INFO : Plotted number of elements +rank0 [2026-10-28 12:35:55,873] INFO : Plotted jets +rank0 [2026-10-28 12:35:56,964] INFO : Plotted jet ratio +rank0 [2026-10-28 12:35:56,998] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:57,224] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:35:57,606] INFO : Plotted number of elements +rank0 [2026-10-28 12:35:59,478] INFO : Plotted jets +rank0 [2026-10-28 12:36:00,591] INFO : Plotted jet ratio +rank0 [2026-10-28 12:36:00,625] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 12:36:00,823] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 12:36:01,239] INFO : Plotted number of elements +rank0 [2026-10-28 12:36:02,636] INFO : Plotted jets +rank0 [2026-10-28 12:36:03,728] INFO : Plotted jet ratio +rank0 [2026-10-28 12:36:14,885] INFO : Step 29100/50000 rank0 | Train Loss: 8.2495 | Train Loss Unweighted: 2.8462 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5490 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1263 | Regression_eta: 0.0096 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6456 | Classification_binary: 2.5490 | Regression_cos_phi: 0.4857 | Regression_energy: 1.7004 | Regression_eta: 0.4015 | Regression_pt: 1.8330 | Regression_sin_phi: 0.6341 | LR: 4.35e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0136 | max_loss: 0.0033 | max_backward: 0.0803 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4998.0 | uss_MB: 4521.3 | child_rss_MB: 3228.4 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:36:15,100] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:25,892] INFO : Step 29200/50000 rank0 | Train Loss: 8.8682 | Train Loss Unweighted: 2.9314 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6038 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1401 | Regression_eta: 0.0103 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6937 | Classification_binary: 2.6038 | Regression_cos_phi: 0.5453 | Regression_energy: 1.8857 | Regression_eta: 0.4320 | Regression_pt: 2.0194 | Regression_sin_phi: 0.6884 | LR: 4.33e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.2680s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0032 | max_backward: 0.0342 | max_optimizer_step: 0.1592 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3202.9 | allocated_MB: 207.0 | reserved_MB: 2650.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:26,125] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:36,779] INFO : Step 29300/50000 rank0 | Train Loss: 8.9416 | Train Loss Unweighted: 2.9850 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6601 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1362 | Regression_eta: 0.0099 | Regression_pt: 0.1392 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7113 | Classification_binary: 2.6601 | Regression_cos_phi: 0.4269 | Regression_energy: 1.8331 | Regression_eta: 0.4146 | Regression_pt: 2.0357 | Regression_sin_phi: 0.8599 | LR: 4.30e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2592s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0188 | max_forward: 0.0152 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.2779 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:36:36,992] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:47,896] INFO : Step 29400/50000 rank0 | Train Loss: 9.1110 | Train Loss Unweighted: 2.9279 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5864 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1477 | Regression_eta: 0.0137 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7179 | Classification_binary: 2.5864 | Regression_cos_phi: 0.5267 | Regression_energy: 1.9887 | Regression_eta: 0.5730 | Regression_pt: 2.0619 | Regression_sin_phi: 0.6563 | LR: 4.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4355s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.3667 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:36:48,109] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:59,017] INFO : Step 29500/50000 rank0 | Train Loss: 8.7958 | Train Loss Unweighted: 2.8937 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5726 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1372 | Regression_eta: 0.0102 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7116 | Classification_binary: 2.5726 | Regression_cos_phi: 0.5096 | Regression_energy: 1.8476 | Regression_eta: 0.4270 | Regression_pt: 1.9544 | Regression_sin_phi: 0.7730 | LR: 4.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3665s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0495 | max_loss: 0.0032 | max_backward: 0.0329 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3725 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3199.3 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:36:59,228] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:10,024] INFO : Step 29600/50000 rank0 | Train Loss: 8.8940 | Train Loss Unweighted: 2.9284 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6100 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1340 | Regression_eta: 0.0103 | Regression_pt: 0.1314 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7340 | Classification_binary: 2.6100 | Regression_cos_phi: 0.5970 | Regression_energy: 1.8039 | Regression_eta: 0.4326 | Regression_pt: 1.9213 | Regression_sin_phi: 0.7951 | LR: 4.22e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0507 | max_backward: 0.0330 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4998.0 | uss_MB: 4521.5 | child_rss_MB: 3203.5 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:10,236] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:21,090] INFO : Step 29700/50000 rank0 | Train Loss: 8.6560 | Train Loss Unweighted: 2.8423 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5228 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1354 | Regression_eta: 0.0111 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7116 | Classification_binary: 2.5228 | Regression_cos_phi: 0.4715 | Regression_energy: 1.8222 | Regression_eta: 0.4639 | Regression_pt: 1.9704 | Regression_sin_phi: 0.6935 | LR: 4.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3097s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0152 | max_backward: 0.0340 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.3636 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4884.2 | uss_MB: 4407.7 | child_rss_MB: 3202.1 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:37:21,301] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:32,223] INFO : Step 29800/50000 rank0 | Train Loss: 8.5682 | Train Loss Unweighted: 2.9332 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6171 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1369 | Regression_eta: 0.0102 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6788 | Classification_binary: 2.6171 | Regression_cos_phi: 0.4548 | Regression_energy: 1.8437 | Regression_eta: 0.4270 | Regression_pt: 1.9455 | Regression_sin_phi: 0.6013 | LR: 4.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.1006 | max_backward: 0.0340 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3711 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4884.2 | uss_MB: 4407.9 | child_rss_MB: 3203.0 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:32,436] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:43,349] INFO : Step 29900/50000 rank0 | Train Loss: 8.5075 | Train Loss Unweighted: 2.9153 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.6042 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1337 | Regression_eta: 0.0100 | Regression_pt: 0.1326 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6098 | Classification_binary: 2.6042 | Regression_cos_phi: 0.4452 | Regression_energy: 1.8006 | Regression_eta: 0.4188 | Regression_pt: 1.9389 | Regression_sin_phi: 0.6901 | LR: 4.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4102s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0141 | max_loss: 0.0888 | max_backward: 0.0338 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.3731 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4884.2 | uss_MB: 4408.0 | child_rss_MB: 3202.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:37:43,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:54,487] INFO : Step 30000/50000 rank0 | Train Loss: 8.3834 | Train Loss Unweighted: 2.8626 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5577 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1311 | Regression_eta: 0.0095 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7365 | Classification_binary: 2.5577 | Regression_cos_phi: 0.4218 | Regression_energy: 1.7654 | Regression_eta: 0.4000 | Regression_pt: 1.8502 | Regression_sin_phi: 0.6517 | LR: 4.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0139 | max_loss: 0.0970 | max_backward: 0.0332 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4884.2 | uss_MB: 4408.2 | child_rss_MB: 3200.7 | allocated_MB: 207.0 | reserved_MB: 2770.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:37:54,701] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:54,718] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 12:37:54,902] INFO : Running validation on rank0 +rank0 [2026-10-28 12:37:54,902] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:37:54,902] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:37:56,024] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 12:37:56,171] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:37:56,303] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:37:56,378] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:37:56,448] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:37:56,515] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:37:56,585] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:37:56,655] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:37:56,769] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:37:56,894] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:37:57,016] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:37:57,089] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:37:57,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:37:57,229] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:37:57,299] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:37:57,369] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:37:57,487] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:37:57,494] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:37:57,497] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:37:57,500] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:37:57,501] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:37:57,503] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:37:57,506] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:37:57,509] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:37:57,512] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:37:57,513] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:37:57,515] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:37:57,516] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:37:57,519] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:37:57,521] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:37:57,524] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:37:57,525] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:38:03,336] INFO : VALIDATION | Step=30000/50000 | Train Loss=8.3834 | Valid Loss=8.6959 | Train Loss Unweighted=2.8626 | Valid Loss Unweighted=2.9131 | Stale=0 | ETA=57.0m +rank0 [2026-10-28 12:38:03,345] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:03,422] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:38:03,426] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:03,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:04,965] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:05,537] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:06,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:06,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:06,818] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:06,821] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:06,875] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:38:06,878] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:06,989] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:08,512] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:09,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:09,600] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:10,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:10,284] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:10,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:38:10,342] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:38:10,345] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:38:10,449] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:38:11,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:38:12,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:38:12,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:38:13,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:38:13,628] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:38:13,636] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:13,885] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:38:14,338] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:15,781] INFO : Plotted jets +rank0 [2026-10-28 12:38:16,877] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:16,909] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:17,133] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:38:17,510] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:19,414] INFO : Plotted jets +rank0 [2026-10-28 12:38:20,525] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:20,557] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 12:38:20,755] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 12:38:21,174] INFO : Plotted number of elements +rank0 [2026-10-28 12:38:22,576] INFO : Plotted jets +rank0 [2026-10-28 12:38:23,670] INFO : Plotted jet ratio +rank0 [2026-10-28 12:38:35,074] INFO : Step 30100/50000 rank0 | Train Loss: 8.3520 | Train Loss Unweighted: 2.8168 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.5002 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1376 | Regression_eta: 0.0084 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6536 | Classification_binary: 2.5002 | Regression_cos_phi: 0.3959 | Regression_energy: 1.8523 | Regression_eta: 0.3539 | Regression_pt: 1.9954 | Regression_sin_phi: 0.6007 | LR: 4.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4125s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0985 | max_loss: 0.0035 | max_backward: 0.0343 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4954.7 | uss_MB: 4478.2 | child_rss_MB: 3199.5 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:35,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:46,183] INFO : Step 30200/50000 rank0 | Train Loss: 8.4129 | Train Loss Unweighted: 2.9089 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.6181 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1190 | Regression_eta: 0.0114 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7579 | Classification_binary: 2.6181 | Regression_cos_phi: 0.4671 | Regression_energy: 1.6014 | Regression_eta: 0.4759 | Regression_pt: 1.7608 | Regression_sin_phi: 0.7318 | LR: 4.06e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4201s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1153 | max_loss: 0.0032 | max_backward: 0.0334 | max_optimizer_step: 0.1243 | max_tensorboard_logging: 0.3688 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4954.7 | uss_MB: 4479.2 | child_rss_MB: 3204.2 | allocated_MB: 207.0 | reserved_MB: 2628.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:38:46,397] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 2% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:38:57,312] INFO : Step 30300/50000 rank0 | Train Loss: 9.2611 | Train Loss Unweighted: 3.0010 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6607 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1457 | Regression_eta: 0.0105 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7100 | Classification_binary: 2.6607 | Regression_cos_phi: 0.5717 | Regression_energy: 1.9615 | Regression_eta: 0.4420 | Regression_pt: 2.0776 | Regression_sin_phi: 0.8376 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4019s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0951 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3688 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4954.7 | uss_MB: 4479.3 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:38:57,523] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:39:08,420] INFO : Step 30400/50000 rank0 | Train Loss: 8.8546 | Train Loss Unweighted: 2.9369 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5946 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1495 | Regression_eta: 0.0085 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7313 | Classification_binary: 2.5946 | Regression_cos_phi: 0.3603 | Regression_energy: 2.0122 | Regression_eta: 0.3556 | Regression_pt: 2.1675 | Regression_sin_phi: 0.6331 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2811s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0588 | max_forward: 0.0165 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1222 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 4954.7 | uss_MB: 4479.4 | child_rss_MB: 3198.9 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:39:08,634] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:19,417] INFO : Step 30500/50000 rank0 | Train Loss: 8.7242 | Train Loss Unweighted: 2.9100 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5866 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1388 | Regression_eta: 0.0090 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.5866 | Regression_cos_phi: 0.5086 | Regression_energy: 1.8690 | Regression_eta: 0.3764 | Regression_pt: 2.0059 | Regression_sin_phi: 0.6860 | LR: 3.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3817s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0632 | max_backward: 0.0337 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3679 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4954.7 | uss_MB: 4479.3 | child_rss_MB: 3196.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:39:19,629] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:30,646] INFO : Step 30600/50000 rank0 | Train Loss: 8.8363 | Train Loss Unweighted: 2.9371 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6006 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1461 | Regression_eta: 0.0092 | Regression_pt: 0.1451 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6946 | Classification_binary: 2.6006 | Regression_cos_phi: 0.3565 | Regression_energy: 1.9674 | Regression_eta: 0.3837 | Regression_pt: 2.1226 | Regression_sin_phi: 0.7108 | LR: 3.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0708 | max_loss: 0.0030 | max_backward: 0.0334 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3729 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4955.7 | uss_MB: 4479.9 | child_rss_MB: 3195.7 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:30,860] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:41,657] INFO : Step 30700/50000 rank0 | Train Loss: 8.5080 | Train Loss Unweighted: 2.8687 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5570 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1327 | Regression_eta: 0.0118 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6806 | Classification_binary: 2.5570 | Regression_cos_phi: 0.4581 | Regression_energy: 1.7862 | Regression_eta: 0.4947 | Regression_pt: 1.9145 | Regression_sin_phi: 0.6169 | LR: 3.92e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3653s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0139 | max_loss: 0.0387 | max_backward: 0.0329 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3812 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4954.4 | uss_MB: 4478.3 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:41,872] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:52,558] INFO : Step 30800/50000 rank0 | Train Loss: 8.7070 | Train Loss Unweighted: 2.7677 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4453 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1370 | Regression_eta: 0.0103 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6667 | Classification_binary: 2.4453 | Regression_cos_phi: 0.4769 | Regression_energy: 1.8446 | Regression_eta: 0.4332 | Regression_pt: 1.9881 | Regression_sin_phi: 0.8523 | LR: 3.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2401s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0167 | max_loss: 0.0032 | max_backward: 0.0458 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.2725 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4954.4 | uss_MB: 4479.4 | child_rss_MB: 3221.5 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:39:52,777] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:03,671] INFO : Step 30900/50000 rank0 | Train Loss: 8.6310 | Train Loss Unweighted: 2.9212 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5973 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1403 | Regression_eta: 0.0082 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6778 | Classification_binary: 2.5973 | Regression_cos_phi: 0.4455 | Regression_energy: 1.8883 | Regression_eta: 0.3451 | Regression_pt: 2.0350 | Regression_sin_phi: 0.6419 | LR: 3.87e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.3426s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0161 | max_loss: 0.0032 | max_backward: 0.1393 | max_optimizer_step: 0.1241 | max_tensorboard_logging: 0.2781 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4954.4 | uss_MB: 4479.5 | child_rss_MB: 3222.3 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:40:03,884] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:14,672] INFO : Step 31000/50000 rank0 | Train Loss: 8.2720 | Train Loss Unweighted: 2.8017 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.5101 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1218 | Regression_eta: 0.0094 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6499 | Classification_binary: 2.5101 | Regression_cos_phi: 0.5236 | Regression_energy: 1.6399 | Regression_eta: 0.3960 | Regression_pt: 1.7827 | Regression_sin_phi: 0.7698 | LR: 3.84e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0135 | max_loss: 0.0031 | max_backward: 0.0328 | max_optimizer_step: 0.1930 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4955.4 | uss_MB: 4479.8 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2748.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:40:14,897] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:14,920] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 12:40:15,000] INFO : Running validation on rank0 +rank0 [2026-10-28 12:40:15,000] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:40:15,001] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:40:16,189] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 12:40:16,336] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:40:16,469] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:40:16,544] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:40:16,613] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:40:16,681] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:40:16,750] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:40:16,819] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:40:16,932] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:40:17,057] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:40:17,180] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:40:17,252] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:40:17,321] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:40:17,390] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:40:17,459] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:40:17,528] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:40:17,642] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:40:17,649] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:40:17,651] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:40:17,653] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:40:17,655] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:40:17,656] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:40:17,659] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:40:17,662] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:40:17,664] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:40:17,665] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:40:17,668] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:40:17,669] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:40:17,671] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:40:17,674] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:40:17,676] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:40:17,677] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:40:23,473] INFO : VALIDATION | Step=31000/50000 | Train Loss=8.2720 | Valid Loss=8.6164 | Train Loss Unweighted=2.8017 | Valid Loss Unweighted=2.8884 | Stale=0 | ETA=53.8m +rank0 [2026-10-28 12:40:23,482] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:23,544] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:40:23,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:23,720] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:25,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:25,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:26,368] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:27,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:27,159] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:40:27,162] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:27,218] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:40:27,220] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:27,392] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:28,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:29,430] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:30,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:30,599] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:30,750] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:40:30,753] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:40:30,807] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:40:30,810] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:40:30,923] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:40:32,363] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:40:32,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:40:33,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:40:34,086] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:40:34,246] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:40:34,253] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:34,507] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:40:34,961] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:36,405] INFO : Plotted jets +rank0 [2026-10-28 12:40:37,504] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:37,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:37,757] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:40:38,139] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:40,043] INFO : Plotted jets +rank0 [2026-10-28 12:40:41,163] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:41,211] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 12:40:41,413] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 12:40:41,828] INFO : Plotted number of elements +rank0 [2026-10-28 12:40:43,226] INFO : Plotted jets +rank0 [2026-10-28 12:40:44,335] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:55,629] INFO : Step 31100/50000 rank0 | Train Loss: 8.1563 | Train Loss Unweighted: 2.7358 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.4528 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1162 | Regression_eta: 0.0113 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7347 | Classification_binary: 2.4528 | Regression_cos_phi: 0.5140 | Regression_energy: 1.5649 | Regression_eta: 0.4724 | Regression_pt: 1.6835 | Regression_sin_phi: 0.7340 | LR: 3.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4021s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0176 | max_loss: 0.0611 | max_backward: 0.0386 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3830 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5027.3 | uss_MB: 4555.2 | child_rss_MB: 3197.3 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:40:55,845] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:06,747] INFO : Step 31200/50000 rank0 | Train Loss: 8.0745 | Train Loss Unweighted: 2.8105 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5270 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1184 | Regression_eta: 0.0097 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6720 | Classification_binary: 2.5270 | Regression_cos_phi: 0.4353 | Regression_energy: 1.5944 | Regression_eta: 0.4063 | Regression_pt: 1.7362 | Regression_sin_phi: 0.7033 | LR: 3.79e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3788s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0372 | max_backward: 0.0330 | max_optimizer_step: 0.1267 | max_tensorboard_logging: 0.3864 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5028.3 | uss_MB: 4554.1 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:41:06,962] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:17,845] INFO : Step 31300/50000 rank0 | Train Loss: 9.0935 | Train Loss Unweighted: 2.9418 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.6057 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1404 | Regression_eta: 0.0120 | Regression_pt: 0.1415 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7996 | Classification_binary: 2.6057 | Regression_cos_phi: 0.5264 | Regression_energy: 1.8904 | Regression_eta: 0.5035 | Regression_pt: 2.0701 | Regression_sin_phi: 0.6977 | LR: 3.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3783s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0136 | max_loss: 0.0545 | max_backward: 0.0328 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5028.3 | uss_MB: 4554.1 | child_rss_MB: 3194.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:41:18,055] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:28,729] INFO : Step 31400/50000 rank0 | Train Loss: 8.2347 | Train Loss Unweighted: 2.7348 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4126 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1428 | Regression_eta: 0.0082 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5965 | Classification_binary: 2.4126 | Regression_cos_phi: 0.3352 | Regression_energy: 1.9223 | Regression_eta: 0.3439 | Regression_pt: 2.0506 | Regression_sin_phi: 0.5737 | LR: 3.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.2626s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0032 | max_backward: 0.0722 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.2749 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 5028.3 | uss_MB: 4554.3 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:41:28,943] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:39,504] INFO : Step 31500/50000 rank0 | Train Loss: 8.8117 | Train Loss Unweighted: 2.9383 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.6218 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1321 | Regression_eta: 0.0096 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6799 | Classification_binary: 2.6218 | Regression_cos_phi: 0.5327 | Regression_energy: 1.7780 | Regression_eta: 0.4042 | Regression_pt: 1.9671 | Regression_sin_phi: 0.8280 | LR: 3.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2541s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0357 | max_forward: 0.0191 | max_loss: 0.0032 | max_backward: 0.0389 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.2744 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 5029.3 | uss_MB: 4556.8 | child_rss_MB: 3194.1 | allocated_MB: 210.7 | reserved_MB: 2458.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 12:41:39,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 131.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:50,634] INFO : Step 31600/50000 rank0 | Train Loss: 9.2268 | Train Loss Unweighted: 2.9088 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5622 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1471 | Regression_eta: 0.0127 | Regression_pt: 0.1457 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7541 | Classification_binary: 2.5622 | Regression_cos_phi: 0.5048 | Regression_energy: 1.9802 | Regression_eta: 0.5331 | Regression_pt: 2.1303 | Regression_sin_phi: 0.7621 | LR: 3.69e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3514s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0415 | max_loss: 0.0033 | max_backward: 0.0339 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3701 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5029.3 | uss_MB: 4554.8 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:41:50,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:01,632] INFO : Step 31700/50000 rank0 | Train Loss: 8.4432 | Train Loss Unweighted: 2.7798 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4590 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1382 | Regression_eta: 0.0097 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.4590 | Regression_cos_phi: 0.3761 | Regression_energy: 1.8610 | Regression_eta: 0.4085 | Regression_pt: 2.0177 | Regression_sin_phi: 0.6509 | LR: 3.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3875s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0142 | max_loss: 0.0777 | max_backward: 0.0324 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5027.2 | uss_MB: 4553.3 | child_rss_MB: 3189.9 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:01,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:12,748] INFO : Step 31800/50000 rank0 | Train Loss: 8.6059 | Train Loss Unweighted: 2.8804 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5719 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1315 | Regression_eta: 0.0116 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6855 | Classification_binary: 2.5719 | Regression_cos_phi: 0.4474 | Regression_energy: 1.7705 | Regression_eta: 0.4882 | Regression_pt: 1.8570 | Regression_sin_phi: 0.7854 | LR: 3.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4278s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0152 | max_loss: 0.0745 | max_backward: 0.0338 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0203 | Memory: rss_MB: 5029.2 | uss_MB: 4554.3 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:12,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:23,824] INFO : Step 31900/50000 rank0 | Train Loss: 8.1421 | Train Loss Unweighted: 2.7852 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4828 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1308 | Regression_eta: 0.0083 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6229 | Classification_binary: 2.4828 | Regression_cos_phi: 0.3708 | Regression_energy: 1.7604 | Regression_eta: 0.3494 | Regression_pt: 1.8984 | Regression_sin_phi: 0.6573 | LR: 3.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4003s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0578 | max_backward: 0.0337 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5029.2 | uss_MB: 4554.4 | child_rss_MB: 3187.3 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:42:24,037] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:34,932] INFO : Step 32000/50000 rank0 | Train Loss: 8.6202 | Train Loss Unweighted: 2.7881 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4681 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1349 | Regression_eta: 0.0087 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6916 | Classification_binary: 2.4681 | Regression_cos_phi: 0.5949 | Regression_energy: 1.8166 | Regression_eta: 0.3667 | Regression_pt: 1.9932 | Regression_sin_phi: 0.6890 | LR: 3.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3684s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0553 | max_backward: 0.0331 | max_optimizer_step: 0.1240 | max_tensorboard_logging: 0.3717 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5030.2 | uss_MB: 4554.4 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2458.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:42:35,144] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:35,162] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 12:42:35,347] INFO : Running validation on rank0 +rank0 [2026-10-28 12:42:35,348] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:42:35,348] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:42:36,428] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 12:42:36,572] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:42:36,705] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:42:36,778] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:42:36,848] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:42:36,915] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:42:36,984] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:42:37,053] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:42:37,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:42:37,297] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:42:37,429] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:42:37,500] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:42:37,569] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:42:37,637] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:42:37,708] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:42:37,777] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:42:37,894] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:42:37,901] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:42:37,904] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:42:37,907] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:42:37,908] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:42:37,911] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:42:37,913] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:42:37,916] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:42:37,917] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:42:37,920] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:42:37,922] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:42:37,925] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:42:37,926] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:42:37,928] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:42:37,931] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:42:37,932] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:42:43,812] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.6202 | Valid Loss=8.5531 | Train Loss Unweighted=2.7881 | Valid Loss Unweighted=2.8658 | Stale=0 | ETA=50.7m +rank0 [2026-10-28 12:42:43,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:43,882] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:42:43,886] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:44,055] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:45,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:46,108] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:46,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:47,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:47,358] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:42:47,362] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:47,418] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:42:47,421] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:47,527] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:49,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:49,741] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:50,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:50,823] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:50,994] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:42:50,998] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:42:51,050] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:42:51,053] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:42:51,169] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:42:52,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:42:53,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:42:53,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:42:54,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:42:54,422] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:42:54,429] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:54,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:42:55,138] INFO : Plotted number of elements +rank0 [2026-10-28 12:42:56,585] INFO : Plotted jets +rank0 [2026-10-28 12:42:57,685] INFO : Plotted jet ratio +rank0 [2026-10-28 12:42:57,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:57,944] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:42:58,321] INFO : Plotted number of elements +rank0 [2026-10-28 12:43:00,214] INFO : Plotted jets +rank0 [2026-10-28 12:43:01,288] INFO : Plotted jet ratio +rank0 [2026-10-28 12:43:01,322] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 12:43:01,523] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 12:43:01,942] INFO : Plotted number of elements +rank0 [2026-10-28 12:43:03,342] INFO : Plotted jets +rank0 [2026-10-28 12:43:04,456] INFO : Plotted jet ratio +rank0 [2026-10-28 12:43:15,866] INFO : Step 32100/50000 rank0 | Train Loss: 8.1447 | Train Loss Unweighted: 2.7034 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4024 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1280 | Regression_eta: 0.0085 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6635 | Classification_binary: 2.4024 | Regression_cos_phi: 0.4751 | Regression_energy: 1.7238 | Regression_eta: 0.3549 | Regression_pt: 1.8719 | Regression_sin_phi: 0.6531 | LR: 3.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4624s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1228 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5227.3 | uss_MB: 4751.5 | child_rss_MB: 3191.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:43:16,081] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:26,941] INFO : Step 32200/50000 rank0 | Train Loss: 8.3482 | Train Loss Unweighted: 2.8947 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5831 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1343 | Regression_eta: 0.0081 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6638 | Classification_binary: 2.5831 | Regression_cos_phi: 0.4260 | Regression_energy: 1.8082 | Regression_eta: 0.3377 | Regression_pt: 1.9721 | Regression_sin_phi: 0.5574 | LR: 3.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3587s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0506 | max_loss: 0.0032 | max_backward: 0.0326 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3673 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 5105.7 | uss_MB: 4629.6 | child_rss_MB: 3188.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:43:27,153] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:38,081] INFO : Step 32300/50000 rank0 | Train Loss: 8.6741 | Train Loss Unweighted: 2.9191 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6067 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1325 | Regression_eta: 0.0098 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6723 | Classification_binary: 2.6067 | Regression_cos_phi: 0.4668 | Regression_energy: 1.7836 | Regression_eta: 0.4086 | Regression_pt: 1.9232 | Regression_sin_phi: 0.8128 | LR: 3.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1077 | max_loss: 0.0032 | max_backward: 0.0327 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3710 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:43:38,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:49,083] INFO : Step 32400/50000 rank0 | Train Loss: 8.0517 | Train Loss Unweighted: 2.7593 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4588 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1279 | Regression_eta: 0.0087 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6312 | Classification_binary: 2.4588 | Regression_cos_phi: 0.3688 | Regression_energy: 1.7222 | Regression_eta: 0.3658 | Regression_pt: 1.9156 | Regression_sin_phi: 0.5893 | LR: 3.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3965s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0579 | max_backward: 0.0336 | max_optimizer_step: 0.1265 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3188.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:43:49,298] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:00,302] INFO : Step 32500/50000 rank0 | Train Loss: 8.3953 | Train Loss Unweighted: 2.8386 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5259 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1337 | Regression_eta: 0.0103 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7200 | Classification_binary: 2.5259 | Regression_cos_phi: 0.3624 | Regression_energy: 1.7995 | Regression_eta: 0.4322 | Regression_pt: 1.9476 | Regression_sin_phi: 0.6078 | LR: 3.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3814s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0546 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3745 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3185.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:44:00,513] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:11,116] INFO : Step 32600/50000 rank0 | Train Loss: 8.5675 | Train Loss Unweighted: 2.8502 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5380 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1324 | Regression_eta: 0.0096 | Regression_pt: 0.1320 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6701 | Classification_binary: 2.5380 | Regression_cos_phi: 0.4650 | Regression_energy: 1.7824 | Regression_eta: 0.4039 | Regression_pt: 1.9305 | Regression_sin_phi: 0.7777 | LR: 3.43e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.2591s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0162 | max_loss: 0.0033 | max_backward: 0.0458 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.2775 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3214.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:44:11,329] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:22,439] INFO : Step 32700/50000 rank0 | Train Loss: 8.4819 | Train Loss Unweighted: 2.7991 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4813 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1364 | Regression_eta: 0.0092 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6560 | Classification_binary: 2.4813 | Regression_cos_phi: 0.4346 | Regression_energy: 1.8357 | Regression_eta: 0.3863 | Regression_pt: 1.9907 | Regression_sin_phi: 0.6973 | LR: 3.41e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4525s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.1094 | max_backward: 0.0331 | max_optimizer_step: 0.1264 | max_tensorboard_logging: 0.3772 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5105.7 | uss_MB: 4630.1 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:44:22,651] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:33,536] INFO : Step 32800/50000 rank0 | Train Loss: 8.8493 | Train Loss Unweighted: 2.8964 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5637 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1432 | Regression_eta: 0.0095 | Regression_pt: 0.1423 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6790 | Classification_binary: 2.5637 | Regression_cos_phi: 0.4549 | Regression_energy: 1.9273 | Regression_eta: 0.3978 | Regression_pt: 2.0807 | Regression_sin_phi: 0.7458 | LR: 3.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3578s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0191 | max_backward: 0.0336 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5105.7 | uss_MB: 4629.8 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:33,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:44,831] INFO : Step 32900/50000 rank0 | Train Loss: 8.4216 | Train Loss Unweighted: 2.8324 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5215 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1334 | Regression_eta: 0.0106 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6373 | Classification_binary: 2.5215 | Regression_cos_phi: 0.4236 | Regression_energy: 1.7965 | Regression_eta: 0.4446 | Regression_pt: 1.9282 | Regression_sin_phi: 0.6699 | LR: 3.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4653s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0924 | max_loss: 0.0033 | max_backward: 0.0338 | max_optimizer_step: 0.1259 | max_tensorboard_logging: 0.4142 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 5105.7 | uss_MB: 4629.9 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:45,044] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:55,850] INFO : Step 33000/50000 rank0 | Train Loss: 8.6157 | Train Loss Unweighted: 2.8881 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5682 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1352 | Regression_eta: 0.0095 | Regression_pt: 0.1379 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6700 | Classification_binary: 2.5682 | Regression_cos_phi: 0.5430 | Regression_energy: 1.8202 | Regression_eta: 0.3972 | Regression_pt: 2.0167 | Regression_sin_phi: 0.6004 | LR: 3.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4706s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0147 | max_loss: 0.0663 | max_backward: 0.0335 | max_optimizer_step: 0.1258 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5106.7 | uss_MB: 4629.9 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 12:44:56,063] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:56,163] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 12:44:56,246] INFO : Running validation on rank0 +rank0 [2026-10-28 12:44:56,247] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:44:56,247] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:44:57,336] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 12:44:57,483] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:44:57,616] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:44:57,691] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:44:57,761] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:44:57,828] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:44:57,898] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:44:57,968] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:44:58,082] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:44:58,208] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:44:58,330] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:44:58,403] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:44:58,474] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:44:58,543] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:44:58,613] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:44:58,682] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:44:58,778] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:44:58,785] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:44:58,788] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:44:58,790] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:44:58,791] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:44:58,794] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:44:58,797] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:44:58,799] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:44:58,800] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:44:58,804] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:44:58,806] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:44:58,809] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:44:58,812] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:44:58,813] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:44:58,816] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:44:58,819] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:45:04,830] INFO : VALIDATION | Step=33000/50000 | Train Loss=8.6157 | Valid Loss=8.5124 | Train Loss Unweighted=2.8881 | Valid Loss Unweighted=2.8510 | Stale=0 | ETA=47.7m +rank0 [2026-10-28 12:45:04,839] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:04,902] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:45:04,906] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:05,077] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:06,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:07,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:07,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:08,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:08,454] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:45:08,457] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:08,513] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:45:08,516] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:08,625] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:09,985] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:10,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:11,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:11,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:11,855] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:45:11,858] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:45:11,911] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:45:11,915] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:45:12,092] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:45:13,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:45:13,953] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:45:14,547] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:45:15,059] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:45:15,241] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:45:15,248] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:15,499] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:45:15,955] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:17,411] INFO : Plotted jets +rank0 [2026-10-28 12:45:18,507] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:18,539] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:18,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:45:19,144] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:21,083] INFO : Plotted jets +rank0 [2026-10-28 12:45:22,195] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:22,227] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 12:45:22,430] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 12:45:22,851] INFO : Plotted number of elements +rank0 [2026-10-28 12:45:24,250] INFO : Plotted jets +rank0 [2026-10-28 12:45:25,341] INFO : Plotted jet ratio +rank0 [2026-10-28 12:45:36,505] INFO : Step 33100/50000 rank0 | Train Loss: 8.0103 | Train Loss Unweighted: 2.7976 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4994 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1296 | Regression_eta: 0.0078 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6668 | Classification_binary: 2.4994 | Regression_cos_phi: 0.3735 | Regression_energy: 1.7447 | Regression_eta: 0.3260 | Regression_pt: 1.8649 | Regression_sin_phi: 0.5350 | LR: 3.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4105s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0154 | max_loss: 0.0248 | max_backward: 0.0340 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5301.6 | uss_MB: 4826.8 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:45:36,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:47,709] INFO : Step 33200/50000 rank0 | Train Loss: 8.4517 | Train Loss Unweighted: 2.8167 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5055 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1327 | Regression_eta: 0.0098 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6728 | Classification_binary: 2.5055 | Regression_cos_phi: 0.3991 | Regression_energy: 1.7863 | Regression_eta: 0.4097 | Regression_pt: 1.9329 | Regression_sin_phi: 0.7453 | LR: 3.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3944s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0826 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3621 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5181.3 | uss_MB: 4704.6 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:45:47,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:58,745] INFO : Step 33300/50000 rank0 | Train Loss: 8.7764 | Train Loss Unweighted: 2.8352 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4978 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1451 | Regression_eta: 0.0088 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.4978 | Regression_cos_phi: 0.4637 | Regression_energy: 1.9537 | Regression_eta: 0.3676 | Regression_pt: 2.1397 | Regression_sin_phi: 0.6654 | LR: 3.26e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3594s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0153 | max_loss: 0.0060 | max_backward: 0.0326 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5181.3 | uss_MB: 4704.7 | child_rss_MB: 3188.2 | allocated_MB: 207.0 | reserved_MB: 2684.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:45:58,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:09,981] INFO : Step 33400/50000 rank0 | Train Loss: 7.8486 | Train Loss Unweighted: 2.6863 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4072 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1178 | Regression_eta: 0.0086 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6466 | Classification_binary: 2.4072 | Regression_cos_phi: 0.4361 | Regression_energy: 1.5865 | Regression_eta: 0.3608 | Regression_pt: 1.7070 | Regression_sin_phi: 0.7045 | LR: 3.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4284s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1265 | max_tensorboard_logging: 0.3811 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3189.3 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:46:10,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:21,073] INFO : Step 33500/50000 rank0 | Train Loss: 9.3423 | Train Loss Unweighted: 2.9498 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.5952 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1509 | Regression_eta: 0.0110 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7289 | Classification_binary: 2.5952 | Regression_cos_phi: 0.5781 | Regression_energy: 2.0310 | Regression_eta: 0.4598 | Regression_pt: 2.2149 | Regression_sin_phi: 0.7344 | LR: 3.21e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1210 | max_loss: 0.0033 | max_backward: 0.0336 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3738 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:46:21,288] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:31,859] INFO : Step 33600/50000 rank0 | Train Loss: 8.4350 | Train Loss Unweighted: 2.8977 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5968 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1263 | Regression_eta: 0.0103 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6837 | Classification_binary: 2.5968 | Regression_cos_phi: 0.4473 | Regression_energy: 1.6997 | Regression_eta: 0.4322 | Regression_pt: 1.8549 | Regression_sin_phi: 0.7204 | LR: 3.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.2608s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0155 | max_loss: 0.0032 | max_backward: 0.0442 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2815 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5181.3 | uss_MB: 4705.1 | child_rss_MB: 3214.8 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:32,074] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:42,991] INFO : Step 33700/50000 rank0 | Train Loss: 8.5533 | Train Loss Unweighted: 2.7539 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4250 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1418 | Regression_eta: 0.0101 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6485 | Classification_binary: 2.4250 | Regression_cos_phi: 0.3751 | Regression_energy: 1.9095 | Regression_eta: 0.4215 | Regression_pt: 2.0802 | Regression_sin_phi: 0.6935 | LR: 3.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3645s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0033 | max_backward: 0.1351 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5181.3 | uss_MB: 4705.3 | child_rss_MB: 3220.2 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:43,205] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:54,092] INFO : Step 33800/50000 rank0 | Train Loss: 8.5593 | Train Loss Unweighted: 2.9054 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1350 | Regression_eta: 0.0087 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6742 | Classification_binary: 2.5905 | Regression_cos_phi: 0.5083 | Regression_energy: 1.8180 | Regression_eta: 0.3639 | Regression_pt: 1.9559 | Regression_sin_phi: 0.6485 | LR: 3.14e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3131s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0141 | max_loss: 0.0033 | max_backward: 0.0830 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5179.2 | uss_MB: 4704.5 | child_rss_MB: 3221.5 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:46:54,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:05,419] INFO : Step 33900/50000 rank0 | Train Loss: 9.2130 | Train Loss Unweighted: 2.9121 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5553 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1540 | Regression_eta: 0.0094 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7158 | Classification_binary: 2.5553 | Regression_cos_phi: 0.4338 | Regression_energy: 2.0734 | Regression_eta: 0.3927 | Regression_pt: 2.2581 | Regression_sin_phi: 0.7838 | LR: 3.11e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4832s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.1120 | max_backward: 0.0328 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3878 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5179.2 | uss_MB: 4704.9 | child_rss_MB: 3191.6 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:47:05,633] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:16,512] INFO : Step 34000/50000 rank0 | Train Loss: 8.8382 | Train Loss Unweighted: 2.8593 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5243 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1442 | Regression_eta: 0.0115 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6786 | Classification_binary: 2.5243 | Regression_cos_phi: 0.4835 | Regression_energy: 1.9418 | Regression_eta: 0.4823 | Regression_pt: 2.0784 | Regression_sin_phi: 0.6492 | LR: 3.09e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.4454s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0942 | max_backward: 0.0337 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3838 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5179.2 | uss_MB: 4705.0 | child_rss_MB: 3188.4 | allocated_MB: 207.0 | reserved_MB: 2732.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:47:16,727] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:47:16,745] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 12:47:16,934] INFO : Running validation on rank0 +rank0 [2026-10-28 12:47:16,934] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:47:16,934] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:47:18,017] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 12:47:18,162] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:47:18,295] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:47:18,370] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:47:18,441] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:47:18,508] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:47:18,578] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:47:18,647] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:47:18,759] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:47:18,892] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:47:19,025] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:47:19,097] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:47:19,168] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:47:19,237] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:47:19,307] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:47:19,377] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:47:19,494] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:47:19,501] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:47:19,504] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:47:19,508] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:47:19,510] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:47:19,511] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:47:19,514] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:47:19,517] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:47:19,519] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:47:19,521] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:47:19,523] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:47:19,526] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:47:19,527] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:47:19,530] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:47:19,532] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:47:19,533] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:47:25,403] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.8382 | Valid Loss=8.5099 | Train Loss Unweighted=2.8593 | Valid Loss Unweighted=2.8440 | Stale=0 | ETA=44.6m +rank0 [2026-10-28 12:47:25,412] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:25,473] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:47:25,477] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:25,654] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:27,136] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:27,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:28,386] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:28,944] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:29,098] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:47:29,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:29,157] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:47:29,160] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:29,277] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:30,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:31,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:32,080] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:32,620] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:32,783] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:47:32,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:47:32,839] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:47:32,842] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:47:32,957] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:47:34,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:47:34,940] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:47:35,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:47:36,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:47:36,203] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:47:36,210] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:36,463] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:47:36,917] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:38,373] INFO : Plotted jets +rank0 [2026-10-28 12:47:39,478] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:39,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:39,744] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:47:40,123] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:42,027] INFO : Plotted jets +rank0 [2026-10-28 12:47:43,150] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:43,184] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 12:47:43,384] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 12:47:43,805] INFO : Plotted number of elements +rank0 [2026-10-28 12:47:45,206] INFO : Plotted jets +rank0 [2026-10-28 12:47:46,323] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:57,577] INFO : Step 34100/50000 rank0 | Train Loss: 7.9387 | Train Loss Unweighted: 2.7878 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5141 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1132 | Regression_eta: 0.0093 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6896 | Classification_binary: 2.5141 | Regression_cos_phi: 0.4258 | Regression_energy: 1.5236 | Regression_eta: 0.3888 | Regression_pt: 1.6668 | Regression_sin_phi: 0.7301 | LR: 3.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3735s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1037 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0327 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5372.6 | uss_MB: 4899.7 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2724.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:47:57,790] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:08,706] INFO : Step 34200/50000 rank0 | Train Loss: 8.3928 | Train Loss Unweighted: 2.7651 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4633 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1270 | Regression_eta: 0.0095 | Regression_pt: 0.1247 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7008 | Classification_binary: 2.4633 | Regression_cos_phi: 0.5966 | Regression_energy: 1.7101 | Regression_eta: 0.4001 | Regression_pt: 1.8244 | Regression_sin_phi: 0.6974 | LR: 3.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4829s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1211 | max_loss: 0.0039 | max_backward: 0.0335 | max_optimizer_step: 0.1277 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5252.1 | uss_MB: 4778.4 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2724.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:48:08,920] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:19,803] INFO : Step 34300/50000 rank0 | Train Loss: 8.7918 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.5171 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1334 | Regression_eta: 0.0118 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6621 | Classification_binary: 2.5171 | Regression_cos_phi: 0.5253 | Regression_energy: 1.7960 | Regression_eta: 0.4947 | Regression_pt: 1.9439 | Regression_sin_phi: 0.8529 | LR: 3.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4180s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0768 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3685 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5252.1 | uss_MB: 4779.1 | child_rss_MB: 3195.6 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:20,015] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:31,018] INFO : Step 34400/50000 rank0 | Train Loss: 8.8160 | Train Loss Unweighted: 2.8325 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.5131 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1329 | Regression_eta: 0.0134 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7799 | Classification_binary: 2.5131 | Regression_cos_phi: 0.5549 | Regression_energy: 1.7887 | Regression_eta: 0.5611 | Regression_pt: 1.9173 | Regression_sin_phi: 0.7009 | LR: 2.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0775 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1365 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.1 | uss_MB: 4779.1 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:48:31,236] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:42,178] INFO : Step 34500/50000 rank0 | Train Loss: 7.7804 | Train Loss Unweighted: 2.7222 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4452 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1170 | Regression_eta: 0.0087 | Regression_pt: 0.1166 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6659 | Classification_binary: 2.4452 | Regression_cos_phi: 0.3808 | Regression_energy: 1.5745 | Regression_eta: 0.3644 | Regression_pt: 1.7046 | Regression_sin_phi: 0.6451 | LR: 2.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4601s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1203 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3666 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.1 | uss_MB: 4779.5 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:48:42,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:53,383] INFO : Step 34600/50000 rank0 | Train Loss: 9.1739 | Train Loss Unweighted: 2.7992 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4352 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1588 | Regression_eta: 0.0108 | Regression_pt: 0.1582 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6283 | Classification_binary: 2.4352 | Regression_cos_phi: 0.4250 | Regression_energy: 2.1376 | Regression_eta: 0.4529 | Regression_pt: 2.3136 | Regression_sin_phi: 0.7813 | LR: 2.95e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5072s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1213 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3871 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5252.1 | uss_MB: 4779.7 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:48:53,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:04,475] INFO : Step 34700/50000 rank0 | Train Loss: 8.9338 | Train Loss Unweighted: 2.8133 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4779 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1441 | Regression_eta: 0.0119 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6721 | Classification_binary: 2.4779 | Regression_cos_phi: 0.5360 | Regression_energy: 1.9397 | Regression_eta: 0.4988 | Regression_pt: 2.0504 | Regression_sin_phi: 0.7589 | LR: 2.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4674s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1179 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1278 | max_tensorboard_logging: 0.3719 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5252.1 | uss_MB: 4779.7 | child_rss_MB: 3195.4 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:49:04,686] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:15,499] INFO : Step 34800/50000 rank0 | Train Loss: 9.0514 | Train Loss Unweighted: 2.8334 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4785 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1548 | Regression_eta: 0.0091 | Regression_pt: 0.1537 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6535 | Classification_binary: 2.4785 | Regression_cos_phi: 0.4839 | Regression_energy: 2.0840 | Regression_eta: 0.3802 | Regression_pt: 2.2486 | Regression_sin_phi: 0.7227 | LR: 2.90e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4164s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0157 | max_loss: 0.0316 | max_backward: 0.0337 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5252.3 | uss_MB: 4778.1 | child_rss_MB: 3194.1 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:49:15,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:26,694] INFO : Step 34900/50000 rank0 | Train Loss: 8.5078 | Train Loss Unweighted: 2.8266 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5090 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1377 | Regression_eta: 0.0090 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6944 | Classification_binary: 2.5090 | Regression_cos_phi: 0.4532 | Regression_energy: 1.8541 | Regression_eta: 0.3753 | Regression_pt: 1.9559 | Regression_sin_phi: 0.6659 | LR: 2.88e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4016s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0481 | max_loss: 0.0033 | max_backward: 0.0329 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3708 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5252.3 | uss_MB: 4779.4 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:49:26,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:37,697] INFO : Step 35000/50000 rank0 | Train Loss: 8.2978 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4251 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1321 | Regression_eta: 0.0088 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6821 | Classification_binary: 2.4251 | Regression_cos_phi: 0.4411 | Regression_energy: 1.7788 | Regression_eta: 0.3681 | Regression_pt: 1.9509 | Regression_sin_phi: 0.6516 | LR: 2.85e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3963s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0158 | max_loss: 0.0288 | max_backward: 0.0338 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5252.3 | uss_MB: 4779.5 | child_rss_MB: 3192.9 | allocated_MB: 207.0 | reserved_MB: 2820.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:49:37,908] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:37,926] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 12:49:38,113] INFO : Running validation on rank0 +rank0 [2026-10-28 12:49:38,114] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:49:38,114] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:49:39,194] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 12:49:39,340] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:49:39,472] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:49:39,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:49:39,618] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:49:39,685] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:49:39,755] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:49:39,825] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:49:39,937] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:49:40,061] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:49:40,185] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:49:40,258] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:49:40,329] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:49:40,397] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:49:40,466] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:49:40,536] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:49:40,653] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:49:40,661] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:49:40,663] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:49:40,664] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:49:40,667] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:49:40,670] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:49:40,673] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:49:40,674] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:49:40,677] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:49:40,680] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:49:40,681] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:49:40,683] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:49:40,686] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:49:40,689] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:49:40,690] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:49:40,693] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:49:46,471] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.2978 | Valid Loss=8.4354 | Train Loss Unweighted=2.7359 | Valid Loss Unweighted=2.8342 | Stale=0 | ETA=41.7m +rank0 [2026-10-28 12:49:46,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:46,543] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:49:46,547] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:46,722] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:48,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:48,675] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:49,293] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:49,852] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:50,025] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:50,029] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:50,086] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:49:50,090] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:49:50,195] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:51,678] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:52,227] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:52,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:53,323] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:53,499] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:53,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:49:53,557] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:49:53,559] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:49:53,675] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:49:55,091] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:49:55,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:49:56,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:49:56,662] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:49:56,845] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:49:56,852] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:57,106] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:49:57,562] INFO : Plotted number of elements +rank0 [2026-10-28 12:49:59,004] INFO : Plotted jets +rank0 [2026-10-28 12:50:00,097] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:00,131] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:00,357] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:50:00,737] INFO : Plotted number of elements +rank0 [2026-10-28 12:50:02,124] INFO : Plotted jets +rank0 [2026-10-28 12:50:03,746] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:03,779] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 12:50:03,981] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 12:50:04,395] INFO : Plotted number of elements +rank0 [2026-10-28 12:50:05,791] INFO : Plotted jets +rank0 [2026-10-28 12:50:06,881] INFO : Plotted jet ratio +rank0 [2026-10-28 12:50:18,098] INFO : Step 35100/50000 rank0 | Train Loss: 8.4273 | Train Loss Unweighted: 2.8285 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5107 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1371 | Regression_eta: 0.0095 | Regression_pt: 0.1372 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6418 | Classification_binary: 2.5107 | Regression_cos_phi: 0.3800 | Regression_energy: 1.8455 | Regression_eta: 0.3965 | Regression_pt: 2.0063 | Regression_sin_phi: 0.6466 | LR: 2.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4616s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0972 | max_loss: 0.0033 | max_backward: 0.0326 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3676 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5443.8 | uss_MB: 4968.9 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:50:18,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:29,246] INFO : Step 35200/50000 rank0 | Train Loss: 8.8541 | Train Loss Unweighted: 2.8656 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5273 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1464 | Regression_eta: 0.0116 | Regression_pt: 0.1440 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6638 | Classification_binary: 2.5273 | Regression_cos_phi: 0.5097 | Regression_energy: 1.9705 | Regression_eta: 0.4859 | Regression_pt: 2.1056 | Regression_sin_phi: 0.5913 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4655s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0573 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.4173 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5327.2 | uss_MB: 4853.0 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:50:29,457] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:40,299] INFO : Step 35300/50000 rank0 | Train Loss: 8.1823 | Train Loss Unweighted: 2.8232 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5327 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1211 | Regression_eta: 0.0090 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7140 | Classification_binary: 2.5327 | Regression_cos_phi: 0.4623 | Regression_energy: 1.6309 | Regression_eta: 0.3752 | Regression_pt: 1.7877 | Regression_sin_phi: 0.6794 | LR: 2.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4224s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0740 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3696 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5328.2 | uss_MB: 4853.6 | child_rss_MB: 3197.5 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:50:40,515] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:51,251] INFO : Step 35400/50000 rank0 | Train Loss: 8.0081 | Train Loss Unweighted: 2.7832 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5029 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1175 | Regression_eta: 0.0099 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7132 | Classification_binary: 2.5029 | Regression_cos_phi: 0.3957 | Regression_energy: 1.5813 | Regression_eta: 0.4170 | Regression_pt: 1.6918 | Regression_sin_phi: 0.7062 | LR: 2.76e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4041s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0138 | max_loss: 0.0033 | max_backward: 0.0353 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3027 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5328.2 | uss_MB: 4852.7 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:50:51,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:02,546] INFO : Step 35500/50000 rank0 | Train Loss: 8.3301 | Train Loss Unweighted: 2.8331 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5204 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1365 | Regression_eta: 0.0075 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6763 | Classification_binary: 2.5204 | Regression_cos_phi: 0.4288 | Regression_energy: 1.8372 | Regression_eta: 0.3155 | Regression_pt: 1.9502 | Regression_sin_phi: 0.6017 | LR: 2.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4533s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0791 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3869 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3198.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:02,760] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:13,655] INFO : Step 35600/50000 rank0 | Train Loss: 9.0884 | Train Loss Unweighted: 2.8962 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.5561 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1433 | Regression_eta: 0.0126 | Regression_pt: 0.1434 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7592 | Classification_binary: 2.5561 | Regression_cos_phi: 0.5283 | Regression_energy: 1.9293 | Regression_eta: 0.5291 | Regression_pt: 2.0978 | Regression_sin_phi: 0.6887 | LR: 2.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4748s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1238 | max_loss: 0.0032 | max_backward: 0.0339 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3684 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3198.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:13,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:24,680] INFO : Step 35700/50000 rank0 | Train Loss: 8.8943 | Train Loss Unweighted: 2.9542 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6287 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1384 | Regression_eta: 0.0106 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.6287 | Regression_cos_phi: 0.4580 | Regression_energy: 1.8637 | Regression_eta: 0.4432 | Regression_pt: 2.0073 | Regression_sin_phi: 0.7853 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4206s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0387 | max_backward: 0.0327 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5328.2 | uss_MB: 4854.2 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:51:24,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:35,881] INFO : Step 35800/50000 rank0 | Train Loss: 8.1360 | Train Loss Unweighted: 2.7699 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4837 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1200 | Regression_eta: 0.0097 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6422 | Classification_binary: 2.4837 | Regression_cos_phi: 0.4477 | Regression_energy: 1.6153 | Regression_eta: 0.4073 | Regression_pt: 1.7441 | Regression_sin_phi: 0.7958 | LR: 2.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4320s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0819 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5328.3 | uss_MB: 4853.1 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:36,092] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:46,975] INFO : Step 35900/50000 rank0 | Train Loss: 8.1288 | Train Loss Unweighted: 2.7615 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4611 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1291 | Regression_eta: 0.0077 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6367 | Classification_binary: 2.4611 | Regression_cos_phi: 0.3908 | Regression_energy: 1.7386 | Regression_eta: 0.3212 | Regression_pt: 1.8848 | Regression_sin_phi: 0.6955 | LR: 2.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0180 | max_loss: 0.0031 | max_backward: 0.0332 | max_optimizer_step: 0.1281 | max_tensorboard_logging: 0.3687 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5329.3 | uss_MB: 4853.7 | child_rss_MB: 3195.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:51:47,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:58,073] INFO : Step 36000/50000 rank0 | Train Loss: 8.0806 | Train Loss Unweighted: 2.7230 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4226 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1285 | Regression_eta: 0.0115 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6575 | Classification_binary: 2.4226 | Regression_cos_phi: 0.3676 | Regression_energy: 1.7303 | Regression_eta: 0.4829 | Regression_pt: 1.8597 | Regression_sin_phi: 0.5601 | LR: 2.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3933s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0500 | max_loss: 0.0033 | max_backward: 0.0335 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.3564 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5329.3 | uss_MB: 4854.0 | child_rss_MB: 3196.8 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:51:58,284] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:58,334] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 12:51:58,416] INFO : Running validation on rank0 +rank0 [2026-10-28 12:51:58,416] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:51:58,416] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:51:59,492] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 12:51:59,636] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:51:59,768] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:51:59,842] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:51:59,912] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:51:59,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:52:00,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:52:00,120] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:52:00,233] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:52:00,358] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:52:00,479] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:52:00,553] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:52:00,623] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:52:00,693] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:52:00,763] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:52:00,832] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:52:00,952] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:52:00,959] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:52:00,961] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:52:00,964] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:52:00,965] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:52:00,968] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:52:00,970] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:52:00,973] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:52:00,974] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:52:00,977] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:52:00,980] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:52:00,983] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:52:00,984] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:52:00,987] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:52:00,989] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:52:00,992] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:52:06,776] INFO : VALIDATION | Step=36000/50000 | Train Loss=8.0806 | Valid Loss=8.3691 | Train Loss Unweighted=2.7230 | Valid Loss Unweighted=2.8026 | Stale=0 | ETA=38.7m +rank0 [2026-10-28 12:52:06,785] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:06,846] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:52:06,850] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:07,020] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:08,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:09,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:09,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:10,527] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:10,693] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:52:10,696] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:10,752] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:52:10,755] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:10,862] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:12,276] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:12,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:13,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:13,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:14,156] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:52:14,159] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:52:14,213] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:52:14,216] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:52:14,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:52:15,813] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:52:16,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:52:16,948] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:52:17,457] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:52:17,591] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:52:17,599] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:17,847] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:52:18,300] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:19,743] INFO : Plotted jets +rank0 [2026-10-28 12:52:20,849] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:20,881] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:21,107] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:52:21,487] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:23,420] INFO : Plotted jets +rank0 [2026-10-28 12:52:24,542] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:24,575] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 12:52:24,778] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 12:52:25,198] INFO : Plotted number of elements +rank0 [2026-10-28 12:52:26,596] INFO : Plotted jets +rank0 [2026-10-28 12:52:27,698] INFO : Plotted jet ratio +rank0 [2026-10-28 12:52:39,010] INFO : Step 36100/50000 rank0 | Train Loss: 8.2667 | Train Loss Unweighted: 2.8254 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5189 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1313 | Regression_eta: 0.0100 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6678 | Classification_binary: 2.5189 | Regression_cos_phi: 0.3708 | Regression_energy: 1.7672 | Regression_eta: 0.4190 | Regression_pt: 1.9177 | Regression_sin_phi: 0.6052 | LR: 2.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4026s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0408 | max_backward: 0.0329 | max_optimizer_step: 0.1297 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5518.9 | uss_MB: 5044.1 | child_rss_MB: 3203.4 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 12:52:39,223] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:50,176] INFO : Step 36200/50000 rank0 | Train Loss: 8.6056 | Train Loss Unweighted: 2.8050 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4892 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1322 | Regression_eta: 0.0135 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6861 | Classification_binary: 2.4892 | Regression_cos_phi: 0.4970 | Regression_energy: 1.7804 | Regression_eta: 0.5679 | Regression_pt: 1.9373 | Regression_sin_phi: 0.6476 | LR: 2.59e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4699s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0611 | max_backward: 0.0336 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5518.9 | uss_MB: 5044.2 | child_rss_MB: 3202.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:52:50,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:01,329] INFO : Step 36300/50000 rank0 | Train Loss: 8.0558 | Train Loss Unweighted: 2.8031 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5098 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1260 | Regression_eta: 0.0086 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6836 | Classification_binary: 2.5098 | Regression_cos_phi: 0.3991 | Regression_energy: 1.6962 | Regression_eta: 0.3595 | Regression_pt: 1.8088 | Regression_sin_phi: 0.5987 | LR: 2.57e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0539 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1290 | max_tensorboard_logging: 0.3512 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5405.1 | uss_MB: 4930.4 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:01,540] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:12,502] INFO : Step 36400/50000 rank0 | Train Loss: 8.3595 | Train Loss Unweighted: 2.8477 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5497 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1252 | Regression_eta: 0.0099 | Regression_pt: 0.1253 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6694 | Classification_binary: 2.5497 | Regression_cos_phi: 0.4667 | Regression_energy: 1.6859 | Regression_eta: 0.4139 | Regression_pt: 1.8322 | Regression_sin_phi: 0.7416 | LR: 2.55e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.3696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0848 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1308 | max_tensorboard_logging: 0.2834 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5405.1 | uss_MB: 4930.5 | child_rss_MB: 3203.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:53:12,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:23,583] INFO : Step 36500/50000 rank0 | Train Loss: 7.7467 | Train Loss Unweighted: 2.7494 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4629 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1231 | Regression_eta: 0.0087 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5979 | Classification_binary: 2.4629 | Regression_cos_phi: 0.3608 | Regression_energy: 1.6578 | Regression_eta: 0.3631 | Regression_pt: 1.8180 | Regression_sin_phi: 0.4862 | LR: 2.52e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3571s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0227 | max_loss: 0.0032 | max_backward: 0.0337 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3639 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5405.1 | uss_MB: 4930.5 | child_rss_MB: 3201.2 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:53:23,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:34,695] INFO : Step 36600/50000 rank0 | Train Loss: 7.8198 | Train Loss Unweighted: 2.7002 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4185 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1193 | Regression_eta: 0.0092 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6493 | Classification_binary: 2.4185 | Regression_cos_phi: 0.4290 | Regression_energy: 1.6067 | Regression_eta: 0.3876 | Regression_pt: 1.7337 | Regression_sin_phi: 0.5949 | LR: 2.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4478s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0995 | max_loss: 0.0033 | max_backward: 0.0382 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3543 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5405.1 | uss_MB: 4933.4 | child_rss_MB: 3201.1 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 12:53:34,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:45,815] INFO : Step 36700/50000 rank0 | Train Loss: 8.2368 | Train Loss Unweighted: 2.8053 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.5013 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1307 | Regression_eta: 0.0094 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6091 | Classification_binary: 2.5013 | Regression_cos_phi: 0.4199 | Regression_energy: 1.7597 | Regression_eta: 0.3958 | Regression_pt: 1.9010 | Regression_sin_phi: 0.6500 | LR: 2.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4496s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1075 | max_loss: 0.0034 | max_backward: 0.0322 | max_optimizer_step: 0.1274 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5405.1 | uss_MB: 4931.4 | child_rss_MB: 3206.6 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:46,029] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:56,957] INFO : Step 36800/50000 rank0 | Train Loss: 8.5373 | Train Loss Unweighted: 2.7573 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4489 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1296 | Regression_eta: 0.0090 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7133 | Classification_binary: 2.4489 | Regression_cos_phi: 0.4195 | Regression_energy: 1.7453 | Regression_eta: 0.3785 | Regression_pt: 1.8887 | Regression_sin_phi: 0.9431 | LR: 2.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4394s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0976 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3657 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5405.1 | uss_MB: 4931.4 | child_rss_MB: 3208.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:53:57,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:08,073] INFO : Step 36900/50000 rank0 | Train Loss: 8.1474 | Train Loss Unweighted: 2.6819 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3718 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1335 | Regression_eta: 0.0107 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6151 | Classification_binary: 2.3718 | Regression_cos_phi: 0.4911 | Regression_energy: 1.7968 | Regression_eta: 0.4479 | Regression_pt: 1.9401 | Regression_sin_phi: 0.4846 | LR: 2.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4086s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0021 | max_forward: 0.0429 | max_loss: 0.0035 | max_backward: 0.0379 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.3635 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5404.9 | uss_MB: 4932.9 | child_rss_MB: 3206.8 | allocated_MB: 210.7 | reserved_MB: 2826.0 | inactive_split_MB: 23.3 +rank0 [2026-10-28 12:54:08,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 47.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:19,153] INFO : Step 37000/50000 rank0 | Train Loss: 8.8223 | Train Loss Unweighted: 2.7968 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4517 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1495 | Regression_eta: 0.0109 | Regression_pt: 0.1491 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6895 | Classification_binary: 2.4517 | Regression_cos_phi: 0.4048 | Regression_energy: 2.0127 | Regression_eta: 0.4579 | Regression_pt: 2.1800 | Regression_sin_phi: 0.6257 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3899s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0661 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.3651 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5404.9 | uss_MB: 4931.2 | child_rss_MB: 3203.1 | allocated_MB: 207.0 | reserved_MB: 2826.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:54:19,365] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:54:19,418] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 12:54:19,503] INFO : Running validation on rank0 +rank0 [2026-10-28 12:54:19,504] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:54:19,504] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:54:20,587] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 12:54:20,731] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:54:20,863] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:54:20,938] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:54:21,008] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:54:21,076] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:54:21,146] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:54:21,215] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:54:21,328] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:54:21,463] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:54:21,596] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:54:21,667] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:54:21,738] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:54:21,806] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:54:21,877] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:54:21,948] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:54:22,045] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:54:22,052] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:54:22,055] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:54:22,057] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:54:22,058] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:54:22,061] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:54:22,064] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:54:22,064] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:54:22,067] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:54:22,069] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:54:22,071] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:54:22,074] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:54:22,075] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:54:22,077] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:54:22,079] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:54:22,081] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:54:27,863] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.8223 | Valid Loss=8.3438 | Train Loss Unweighted=2.7968 | Valid Loss Unweighted=2.7914 | Stale=0 | ETA=35.8m +rank0 [2026-10-28 12:54:27,871] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:27,932] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:54:27,935] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:28,113] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:29,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:30,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:31,013] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:31,564] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:31,723] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:54:31,726] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:31,782] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:54:31,785] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:31,896] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:33,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:34,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:34,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:35,237] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:35,420] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:54:35,423] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:54:35,478] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:54:35,481] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:54:35,606] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:54:37,037] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:54:37,648] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:54:38,169] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:54:38,785] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:54:38,949] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:54:38,956] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:39,215] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:54:39,678] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:41,136] INFO : Plotted jets +rank0 [2026-10-28 12:54:42,256] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:42,297] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:42,522] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:54:42,913] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:44,865] INFO : Plotted jets +rank0 [2026-10-28 12:54:45,991] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:46,027] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:46,236] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:46,652] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:48,067] INFO : Plotted jets +rank0 [2026-10-28 12:54:49,186] INFO : Plotted jet ratio +rank0 [2026-10-28 12:55:00,717] INFO : Step 37100/50000 rank0 | Train Loss: 8.3708 | Train Loss Unweighted: 2.7524 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4462 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1302 | Regression_eta: 0.0103 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6132 | Classification_binary: 2.4462 | Regression_cos_phi: 0.4323 | Regression_energy: 1.7525 | Regression_eta: 0.4319 | Regression_pt: 1.8945 | Regression_sin_phi: 0.8002 | LR: 2.40e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3897s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0479 | max_loss: 0.0034 | max_backward: 0.0341 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3478 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5588.7 | uss_MB: 5116.7 | child_rss_MB: 3200.4 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:00,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:11,852] INFO : Step 37200/50000 rank0 | Train Loss: 8.5016 | Train Loss Unweighted: 2.8241 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5108 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1321 | Regression_eta: 0.0092 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6901 | Classification_binary: 2.5108 | Regression_cos_phi: 0.4730 | Regression_energy: 1.7790 | Regression_eta: 0.3875 | Regression_pt: 1.9615 | Regression_sin_phi: 0.6997 | LR: 2.38e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0235 | max_forward: 0.0140 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.2731 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5588.7 | uss_MB: 5116.7 | child_rss_MB: 3203.7 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:12,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:22,927] INFO : Step 37300/50000 rank0 | Train Loss: 7.5675 | Train Loss Unweighted: 2.6832 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4055 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1194 | Regression_eta: 0.0074 | Regression_pt: 0.1202 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5900 | Classification_binary: 2.4055 | Regression_cos_phi: 0.3526 | Regression_energy: 1.6068 | Regression_eta: 0.3116 | Regression_pt: 1.7573 | Regression_sin_phi: 0.5437 | LR: 2.36e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4533s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1216 | max_loss: 0.0033 | max_backward: 0.0328 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.3496 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5476.1 | uss_MB: 5003.1 | child_rss_MB: 3200.9 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:23,138] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:34,046] INFO : Step 37400/50000 rank0 | Train Loss: 8.0625 | Train Loss Unweighted: 2.6681 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.3753 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1233 | Regression_eta: 0.0093 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7278 | Classification_binary: 2.3753 | Regression_cos_phi: 0.4452 | Regression_energy: 1.6605 | Regression_eta: 0.3898 | Regression_pt: 1.7835 | Regression_sin_phi: 0.6804 | LR: 2.34e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0395 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3606 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5477.1 | uss_MB: 5003.4 | child_rss_MB: 3198.1 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 12:55:34,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:45,164] INFO : Step 37500/50000 rank0 | Train Loss: 7.9934 | Train Loss Unweighted: 2.7050 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4172 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1216 | Regression_eta: 0.0087 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6362 | Classification_binary: 2.4172 | Regression_cos_phi: 0.4415 | Regression_energy: 1.6369 | Regression_eta: 0.3627 | Regression_pt: 1.7792 | Regression_sin_phi: 0.7197 | LR: 2.32e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4065s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0597 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3480 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5477.1 | uss_MB: 5003.4 | child_rss_MB: 3198.2 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:45,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:56,306] INFO : Step 37600/50000 rank0 | Train Loss: 8.5699 | Train Loss Unweighted: 2.7644 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4480 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1332 | Regression_eta: 0.0133 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6847 | Classification_binary: 2.4480 | Regression_cos_phi: 0.4207 | Regression_energy: 1.7927 | Regression_eta: 0.5595 | Regression_pt: 1.9444 | Regression_sin_phi: 0.7200 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3313s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0498 | max_forward: 0.0153 | max_loss: 0.0032 | max_backward: 0.0340 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5477.1 | uss_MB: 5003.7 | child_rss_MB: 3195.9 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:55:56,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:07,376] INFO : Step 37700/50000 rank0 | Train Loss: 8.5468 | Train Loss Unweighted: 2.7946 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4779 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1350 | Regression_eta: 0.0079 | Regression_pt: 0.1342 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6674 | Classification_binary: 2.4779 | Regression_cos_phi: 0.6154 | Regression_energy: 1.8178 | Regression_eta: 0.3326 | Regression_pt: 1.9634 | Regression_sin_phi: 0.6723 | LR: 2.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0180 | max_loss: 0.0033 | max_backward: 0.0380 | max_optimizer_step: 0.1294 | max_tensorboard_logging: 0.3536 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5477.1 | uss_MB: 5005.8 | child_rss_MB: 3196.2 | allocated_MB: 210.7 | reserved_MB: 2754.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 12:56:07,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:18,498] INFO : Step 37800/50000 rank0 | Train Loss: 8.5405 | Train Loss Unweighted: 2.8825 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5696 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1327 | Regression_eta: 0.0106 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6910 | Classification_binary: 2.5696 | Regression_cos_phi: 0.4983 | Regression_energy: 1.7861 | Regression_eta: 0.4454 | Regression_pt: 1.9357 | Regression_sin_phi: 0.6144 | LR: 2.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0850 | max_forward: 0.0139 | max_loss: 0.0033 | max_backward: 0.0332 | max_optimizer_step: 0.1282 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5477.1 | uss_MB: 5003.8 | child_rss_MB: 3195.3 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:56:18,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:29,475] INFO : Step 37900/50000 rank0 | Train Loss: 8.5803 | Train Loss Unweighted: 2.6723 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3514 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1368 | Regression_eta: 0.0089 | Regression_pt: 0.1350 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6635 | Classification_binary: 2.3514 | Regression_cos_phi: 0.5806 | Regression_energy: 1.8414 | Regression_eta: 0.3738 | Regression_pt: 1.9741 | Regression_sin_phi: 0.7956 | LR: 2.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0103 | max_backward: 0.0333 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5475.3 | uss_MB: 5003.1 | child_rss_MB: 3196.1 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:56:29,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:40,581] INFO : Step 38000/50000 rank0 | Train Loss: 8.2256 | Train Loss Unweighted: 2.6641 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3516 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1338 | Regression_eta: 0.0077 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6791 | Classification_binary: 2.3516 | Regression_cos_phi: 0.4444 | Regression_energy: 1.8018 | Regression_eta: 0.3229 | Regression_pt: 1.9667 | Regression_sin_phi: 0.6592 | LR: 2.22e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0149 | max_loss: 0.0221 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3732 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5476.3 | uss_MB: 5003.3 | child_rss_MB: 3196.3 | allocated_MB: 207.0 | reserved_MB: 2754.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:56:40,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:40,811] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 12:56:41,008] INFO : Running validation on rank0 +rank0 [2026-10-28 12:56:41,008] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:56:41,008] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:56:42,079] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 12:56:42,224] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:56:42,356] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:56:42,431] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:56:42,501] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:56:42,569] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:56:42,639] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:56:42,709] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:56:42,822] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:56:42,950] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:56:43,073] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:56:43,146] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:56:43,215] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:56:43,285] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:56:43,355] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:56:43,424] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:56:43,525] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:56:43,531] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:56:43,534] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:56:43,537] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:56:43,538] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:56:43,541] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:56:43,544] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:56:43,547] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:56:43,549] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:56:43,551] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:56:43,553] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:56:43,556] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:56:43,559] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:56:43,560] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:56:43,562] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:56:43,565] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:56:49,474] INFO : VALIDATION | Step=38000/50000 | Train Loss=8.2256 | Valid Loss=8.2885 | Train Loss Unweighted=2.6641 | Valid Loss Unweighted=2.7769 | Stale=0 | ETA=32.9m +rank0 [2026-10-28 12:56:49,484] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:49,545] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:56:49,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:56:49,729] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:51,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:51,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:52,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:52,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:53,183] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:56:53,186] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:53,242] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:56:53,245] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:56:53,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:54,782] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:55,429] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:55,974] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:56:56,511] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:56:56,702] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:56:56,706] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:56:56,761] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:56:56,764] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:56:56,882] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:56:58,416] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:56:59,001] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:56:59,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:57:00,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:57:00,239] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:57:00,246] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:57:00,497] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:57:00,958] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:02,411] INFO : Plotted jets +rank0 [2026-10-28 12:57:03,513] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:03,547] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:57:03,776] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:57:04,153] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:05,556] INFO : Plotted jets +rank0 [2026-10-28 12:57:07,168] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:07,201] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 12:57:07,402] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 12:57:07,824] INFO : Plotted number of elements +rank0 [2026-10-28 12:57:09,228] INFO : Plotted jets +rank0 [2026-10-28 12:57:10,329] INFO : Plotted jet ratio +rank0 [2026-10-28 12:57:21,748] INFO : Step 38100/50000 rank0 | Train Loss: 8.1757 | Train Loss Unweighted: 2.7130 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3970 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1373 | Regression_eta: 0.0082 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6076 | Classification_binary: 2.3970 | Regression_cos_phi: 0.4134 | Regression_energy: 1.8480 | Regression_eta: 0.3434 | Regression_pt: 2.0191 | Regression_sin_phi: 0.5472 | LR: 2.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4477s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1029 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3519 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5668.2 | uss_MB: 5195.1 | child_rss_MB: 3196.0 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:21,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:32,919] INFO : Step 38200/50000 rank0 | Train Loss: 8.1299 | Train Loss Unweighted: 2.6663 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3622 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1294 | Regression_eta: 0.0115 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6721 | Classification_binary: 2.3622 | Regression_cos_phi: 0.3757 | Regression_energy: 1.7418 | Regression_eta: 0.4815 | Regression_pt: 1.8830 | Regression_sin_phi: 0.6136 | LR: 2.18e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3803s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0826 | max_forward: 0.0161 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1339 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5668.2 | uss_MB: 5195.2 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:57:33,132] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:43,673] INFO : Step 38300/50000 rank0 | Train Loss: 8.0930 | Train Loss Unweighted: 2.7598 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4686 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1220 | Regression_eta: 0.0095 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6640 | Classification_binary: 2.4686 | Regression_cos_phi: 0.4356 | Regression_energy: 1.6426 | Regression_eta: 0.3989 | Regression_pt: 1.8068 | Regression_sin_phi: 0.6766 | LR: 2.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0145 | max_loss: 0.0033 | max_backward: 0.0637 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.2838 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.3 | uss_MB: 5076.6 | child_rss_MB: 3227.2 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:43,885] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:54,475] INFO : Step 38400/50000 rank0 | Train Loss: 8.3381 | Train Loss Unweighted: 2.7588 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4569 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1268 | Regression_eta: 0.0096 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6557 | Classification_binary: 2.4569 | Regression_cos_phi: 0.6009 | Regression_energy: 1.7077 | Regression_eta: 0.4036 | Regression_pt: 1.8546 | Regression_sin_phi: 0.6587 | LR: 2.14e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3147s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0257 | max_forward: 0.0156 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.2724 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.3 | uss_MB: 5077.5 | child_rss_MB: 3200.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:57:54,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:05,590] INFO : Step 38500/50000 rank0 | Train Loss: 7.7928 | Train Loss Unweighted: 2.7030 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4219 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1184 | Regression_eta: 0.0095 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6697 | Classification_binary: 2.4219 | Regression_cos_phi: 0.4504 | Regression_energy: 1.5942 | Regression_eta: 0.3991 | Regression_pt: 1.7333 | Regression_sin_phi: 0.5241 | LR: 2.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3491s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0821 | max_forward: 0.0158 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.2760 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5549.3 | uss_MB: 5077.6 | child_rss_MB: 3200.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:58:05,803] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:16,583] INFO : Step 38600/50000 rank0 | Train Loss: 8.5878 | Train Loss Unweighted: 2.8573 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.5402 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1335 | Regression_eta: 0.0109 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6260 | Classification_binary: 2.5402 | Regression_cos_phi: 0.5001 | Regression_energy: 1.7974 | Regression_eta: 0.4560 | Regression_pt: 1.9985 | Regression_sin_phi: 0.6694 | LR: 2.11e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4794s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0134 | max_loss: 0.0983 | max_backward: 0.0317 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5550.3 | uss_MB: 5077.6 | child_rss_MB: 3198.7 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:16,796] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:27,692] INFO : Step 38700/50000 rank0 | Train Loss: 8.8623 | Train Loss Unweighted: 2.7550 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4017 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1528 | Regression_eta: 0.0091 | Regression_pt: 0.1556 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6492 | Classification_binary: 2.4017 | Regression_cos_phi: 0.4975 | Regression_energy: 2.0568 | Regression_eta: 0.3818 | Regression_pt: 2.2749 | Regression_sin_phi: 0.6004 | LR: 2.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4577s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0999 | max_backward: 0.0339 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3656 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5550.3 | uss_MB: 5077.6 | child_rss_MB: 3195.1 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 12:58:27,906] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:38,854] INFO : Step 38800/50000 rank0 | Train Loss: 7.9375 | Train Loss Unweighted: 2.7322 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4476 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1203 | Regression_eta: 0.0084 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6337 | Classification_binary: 2.4476 | Regression_cos_phi: 0.4821 | Regression_energy: 1.6197 | Regression_eta: 0.3511 | Regression_pt: 1.7591 | Regression_sin_phi: 0.6442 | LR: 2.07e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4118s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0154 | max_loss: 0.0339 | max_backward: 0.0335 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 5550.3 | uss_MB: 5077.9 | child_rss_MB: 3194.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:58:39,068] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:49,987] INFO : Step 38900/50000 rank0 | Train Loss: 8.5095 | Train Loss Unweighted: 2.8194 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4957 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1386 | Regression_eta: 0.0091 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6864 | Classification_binary: 2.4957 | Regression_cos_phi: 0.4073 | Regression_energy: 1.8653 | Regression_eta: 0.3828 | Regression_pt: 2.0547 | Regression_sin_phi: 0.6172 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4541s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0156 | max_loss: 0.0789 | max_backward: 0.0340 | max_optimizer_step: 0.1305 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5549.4 | uss_MB: 5076.3 | child_rss_MB: 3193.3 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:58:50,200] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:01,093] INFO : Step 39000/50000 rank0 | Train Loss: 7.8706 | Train Loss Unweighted: 2.6750 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3840 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1242 | Regression_eta: 0.0095 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6758 | Classification_binary: 2.3840 | Regression_cos_phi: 0.3582 | Regression_energy: 1.6719 | Regression_eta: 0.3997 | Regression_pt: 1.8063 | Regression_sin_phi: 0.5746 | LR: 2.03e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4276s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0668 | max_backward: 0.0336 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5549.4 | uss_MB: 5077.1 | child_rss_MB: 3193.5 | allocated_MB: 207.0 | reserved_MB: 2792.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:01,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 124.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:01,579] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 12:59:01,660] INFO : Running validation on rank0 +rank0 [2026-10-28 12:59:01,661] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:59:01,661] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:59:02,732] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 12:59:02,877] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:59:03,011] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:59:03,086] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:59:03,156] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:59:03,226] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:59:03,296] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:59:03,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:59:03,480] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:59:03,614] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:59:03,748] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:59:03,821] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:59:03,891] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:59:03,960] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:59:04,031] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:59:04,102] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:59:04,202] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:59:04,285] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:59:04,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:59:04,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:59:04,294] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:59:04,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:59:04,298] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:59:04,299] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:59:04,301] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:59:04,305] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:59:04,306] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:59:04,308] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:59:04,311] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:59:04,312] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:59:04,314] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:59:04,317] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:59:10,015] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.8706 | Valid Loss=8.2979 | Train Loss Unweighted=2.6750 | Valid Loss Unweighted=2.7831 | Stale=1 | ETA=30.1m +rank0 [2026-10-28 12:59:10,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:10,087] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:59:10,090] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:10,270] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:11,837] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:12,451] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:13,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:13,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:14,000] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:59:14,003] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:14,059] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:59:14,061] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:14,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:15,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:16,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:16,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:17,376] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:17,521] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:59:17,524] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:59:17,579] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:59:17,582] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:59:17,696] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:59:19,055] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:59:19,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:59:20,146] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:59:20,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:59:20,827] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 12:59:20,835] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:21,082] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:59:21,537] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:22,987] INFO : Plotted jets +rank0 [2026-10-28 12:59:24,099] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:24,132] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:24,358] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:59:24,738] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:26,700] INFO : Plotted jets +rank0 [2026-10-28 12:59:27,823] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:27,858] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 12:59:28,059] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 12:59:28,477] INFO : Plotted number of elements +rank0 [2026-10-28 12:59:29,881] INFO : Plotted jets +rank0 [2026-10-28 12:59:30,987] INFO : Plotted jet ratio +rank0 [2026-10-28 12:59:42,129] INFO : Step 39100/50000 rank0 | Train Loss: 7.8538 | Train Loss Unweighted: 2.7071 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4246 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1203 | Regression_eta: 0.0094 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6516 | Classification_binary: 2.4246 | Regression_cos_phi: 0.4269 | Regression_energy: 1.6197 | Regression_eta: 0.3955 | Regression_pt: 1.7250 | Regression_sin_phi: 0.6105 | LR: 2.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0146 | max_loss: 0.0035 | max_backward: 0.1412 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.2754 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5740.2 | uss_MB: 5266.7 | child_rss_MB: 3216.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 12:59:42,339] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:53,498] INFO : Step 39200/50000 rank0 | Train Loss: 8.2073 | Train Loss Unweighted: 2.7348 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4369 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1249 | Regression_eta: 0.0106 | Regression_pt: 0.1256 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6887 | Classification_binary: 2.4369 | Regression_cos_phi: 0.4206 | Regression_energy: 1.6815 | Regression_eta: 0.4450 | Regression_pt: 1.8372 | Regression_sin_phi: 0.6975 | LR: 2.00e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4475s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0147 | max_loss: 0.0647 | max_backward: 0.0335 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3763 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5740.2 | uss_MB: 5266.7 | child_rss_MB: 3191.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 12:59:53,712] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:04,537] INFO : Step 39300/50000 rank0 | Train Loss: 8.6417 | Train Loss Unweighted: 2.8263 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4958 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1422 | Regression_eta: 0.0104 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6156 | Classification_binary: 2.4958 | Regression_cos_phi: 0.4465 | Regression_energy: 1.9138 | Regression_eta: 0.4339 | Regression_pt: 2.1006 | Regression_sin_phi: 0.6355 | LR: 1.98e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4732s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.1067 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5624.3 | uss_MB: 5149.1 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:00:04,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:15,755] INFO : Step 39400/50000 rank0 | Train Loss: 8.6477 | Train Loss Unweighted: 2.8009 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4728 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1399 | Regression_eta: 0.0100 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5995 | Classification_binary: 2.4728 | Regression_cos_phi: 0.4430 | Regression_energy: 1.8833 | Regression_eta: 0.4171 | Regression_pt: 2.0925 | Regression_sin_phi: 0.7395 | LR: 1.96e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0369 | max_backward: 0.0339 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3790 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5624.3 | uss_MB: 5151.1 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:00:15,965] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:26,970] INFO : Step 39500/50000 rank0 | Train Loss: 8.1914 | Train Loss Unweighted: 2.7695 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4630 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1319 | Regression_eta: 0.0075 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6793 | Classification_binary: 2.4630 | Regression_cos_phi: 0.4020 | Regression_energy: 1.7761 | Regression_eta: 0.3150 | Regression_pt: 1.9254 | Regression_sin_phi: 0.6306 | LR: 1.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3816s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1066 | max_forward: 0.0166 | max_loss: 0.0033 | max_backward: 0.0341 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5625.3 | uss_MB: 5151.3 | child_rss_MB: 3190.7 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:00:27,183] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:38,042] INFO : Step 39600/50000 rank0 | Train Loss: 8.0060 | Train Loss Unweighted: 2.7391 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1244 | Regression_eta: 0.0085 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7035 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4564 | Regression_energy: 1.6751 | Regression_eta: 0.3546 | Regression_pt: 1.8428 | Regression_sin_phi: 0.5292 | LR: 1.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4284s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0849 | max_loss: 0.0034 | max_backward: 0.0329 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.3539 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5625.3 | uss_MB: 5151.7 | child_rss_MB: 3191.5 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:00:38,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:49,073] INFO : Step 39700/50000 rank0 | Train Loss: 7.6408 | Train Loss Unweighted: 2.7026 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1105 | Regression_eta: 0.0103 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6782 | Classification_binary: 2.4355 | Regression_cos_phi: 0.4133 | Regression_energy: 1.4877 | Regression_eta: 0.4310 | Regression_pt: 1.6337 | Regression_sin_phi: 0.5615 | LR: 1.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4923s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.1025 | max_backward: 0.0340 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5625.3 | uss_MB: 5151.8 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:00:49,289] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:00,188] INFO : Step 39800/50000 rank0 | Train Loss: 8.5052 | Train Loss Unweighted: 2.8328 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5177 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1337 | Regression_eta: 0.0081 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6570 | Classification_binary: 2.5177 | Regression_cos_phi: 0.5776 | Regression_energy: 1.7997 | Regression_eta: 0.3379 | Regression_pt: 1.9797 | Regression_sin_phi: 0.6356 | LR: 1.89e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4526s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0168 | max_loss: 0.0573 | max_backward: 0.0341 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3797 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5625.3 | uss_MB: 5151.9 | child_rss_MB: 3189.1 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:01:00,401] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:11,393] INFO : Step 39900/50000 rank0 | Train Loss: 8.2951 | Train Loss Unweighted: 2.8066 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5143 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1222 | Regression_eta: 0.0118 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6433 | Classification_binary: 2.5143 | Regression_cos_phi: 0.5178 | Regression_energy: 1.6455 | Regression_eta: 0.4925 | Regression_pt: 1.7683 | Regression_sin_phi: 0.7134 | LR: 1.88e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4369s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1051 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3438 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5624.4 | uss_MB: 5150.4 | child_rss_MB: 3190.2 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:11,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:22,504] INFO : Step 40000/50000 rank0 | Train Loss: 8.7455 | Train Loss Unweighted: 2.8509 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5195 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1413 | Regression_eta: 0.0105 | Regression_pt: 0.1418 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7342 | Classification_binary: 2.5195 | Regression_cos_phi: 0.4531 | Regression_energy: 1.9028 | Regression_eta: 0.4412 | Regression_pt: 2.0738 | Regression_sin_phi: 0.6209 | LR: 1.86e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0922 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1336 | max_tensorboard_logging: 0.3387 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5626.4 | uss_MB: 5151.3 | child_rss_MB: 3188.4 | allocated_MB: 207.0 | reserved_MB: 2786.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:01:22,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:01:22,766] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 13:01:22,848] INFO : Running validation on rank0 +rank0 [2026-10-28 13:01:22,848] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:01:22,848] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:01:23,927] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 13:01:24,071] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:01:24,203] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:01:24,278] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:01:24,348] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:01:24,416] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:01:24,486] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:01:24,557] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:01:24,670] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:01:24,796] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:01:24,920] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:01:24,992] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:01:25,063] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:01:25,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:01:25,201] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:01:25,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:01:25,370] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:01:25,377] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:01:25,379] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:01:25,382] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:01:25,383] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:01:25,386] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:01:25,389] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:01:25,391] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:01:25,392] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:01:25,395] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:01:25,397] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:01:25,400] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:01:25,401] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:01:25,403] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:01:25,406] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:01:25,408] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:01:31,408] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.7455 | Valid Loss=8.2624 | Train Loss Unweighted=2.8509 | Valid Loss Unweighted=2.7656 | Stale=0 | ETA=27.2m +rank0 [2026-10-28 13:01:31,416] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:31,476] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:01:31,479] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:31,658] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:33,105] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:33,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:34,445] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:35,107] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:35,320] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:35,323] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:35,380] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:01:35,383] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:35,515] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:37,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:37,574] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:38,204] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:38,742] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:38,948] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:38,951] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:39,005] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:01:39,009] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:01:39,129] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:40,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:41,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:41,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:42,306] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:42,511] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:42,519] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:42,772] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:43,229] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:44,676] INFO : Plotted jets +rank0 [2026-10-28 13:01:45,789] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:45,823] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:46,049] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:46,429] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:47,831] INFO : Plotted jets +rank0 [2026-10-28 13:01:49,502] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:49,535] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:49,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:50,161] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:51,565] INFO : Plotted jets +rank0 [2026-10-28 13:01:52,669] INFO : Plotted jet ratio +rank0 [2026-10-28 13:02:04,127] INFO : Step 40100/50000 rank0 | Train Loss: 8.3315 | Train Loss Unweighted: 2.7117 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4029 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1321 | Regression_eta: 0.0101 | Regression_pt: 0.1297 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6561 | Classification_binary: 2.4029 | Regression_cos_phi: 0.4964 | Regression_energy: 1.7778 | Regression_eta: 0.4215 | Regression_pt: 1.8969 | Regression_sin_phi: 0.6799 | LR: 1.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3565s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0645 | max_forward: 0.0144 | max_loss: 0.0034 | max_backward: 0.0318 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2751 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5819.0 | uss_MB: 5343.7 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2664.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:04,342] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:15,160] INFO : Step 40200/50000 rank0 | Train Loss: 7.7988 | Train Loss Unweighted: 2.6511 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3757 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1146 | Regression_eta: 0.0102 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6439 | Classification_binary: 2.3757 | Regression_cos_phi: 0.4748 | Regression_energy: 1.5425 | Regression_eta: 0.4255 | Regression_pt: 1.6770 | Regression_sin_phi: 0.6594 | LR: 1.83e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3471s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0136 | max_loss: 0.0038 | max_backward: 0.0330 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3452 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5819.0 | uss_MB: 5343.8 | child_rss_MB: 3194.7 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:02:15,376] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:26,270] INFO : Step 40300/50000 rank0 | Train Loss: 7.9616 | Train Loss Unweighted: 2.7298 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4416 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1220 | Regression_eta: 0.0091 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6653 | Classification_binary: 2.4416 | Regression_cos_phi: 0.4716 | Regression_energy: 1.6422 | Regression_eta: 0.3821 | Regression_pt: 1.7758 | Regression_sin_phi: 0.5830 | LR: 1.81e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3746s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0325 | max_loss: 0.0031 | max_backward: 0.0336 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.3458 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5819.0 | uss_MB: 5343.8 | child_rss_MB: 3194.4 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:02:26,486] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:37,365] INFO : Step 40400/50000 rank0 | Train Loss: 8.1841 | Train Loss Unweighted: 2.7114 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1344 | Regression_eta: 0.0092 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6481 | Classification_binary: 2.4026 | Regression_cos_phi: 0.3488 | Regression_energy: 1.8099 | Regression_eta: 0.3874 | Regression_pt: 1.9191 | Regression_sin_phi: 0.6682 | LR: 1.79e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.3781s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0451 | max_loss: 0.0036 | max_backward: 0.0384 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3373 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5699.7 | uss_MB: 5226.5 | child_rss_MB: 3193.3 | allocated_MB: 210.7 | reserved_MB: 2760.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:02:37,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:48,501] INFO : Step 40500/50000 rank0 | Train Loss: 8.6153 | Train Loss Unweighted: 2.7778 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4537 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1377 | Regression_eta: 0.0107 | Regression_pt: 0.1374 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7101 | Classification_binary: 2.4537 | Regression_cos_phi: 0.5199 | Regression_energy: 1.8539 | Regression_eta: 0.4491 | Regression_pt: 2.0095 | Regression_sin_phi: 0.6191 | LR: 1.78e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3105s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0347 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1320 | max_tensorboard_logging: 0.2727 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5700.7 | uss_MB: 5224.8 | child_rss_MB: 3193.9 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:02:48,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:59,573] INFO : Step 40600/50000 rank0 | Train Loss: 8.0934 | Train Loss Unweighted: 2.7617 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4632 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1271 | Regression_eta: 0.0082 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7013 | Classification_binary: 2.4632 | Regression_cos_phi: 0.4043 | Regression_energy: 1.7116 | Regression_eta: 0.3442 | Regression_pt: 1.8643 | Regression_sin_phi: 0.6045 | LR: 1.76e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4132s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0701 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3405 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5700.7 | uss_MB: 5224.8 | child_rss_MB: 3191.9 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:02:59,783] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:10,671] INFO : Step 40700/50000 rank0 | Train Loss: 7.5546 | Train Loss Unweighted: 2.6525 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.3714 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1216 | Regression_eta: 0.0063 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5883 | Classification_binary: 2.3714 | Regression_cos_phi: 0.3792 | Regression_energy: 1.6371 | Regression_eta: 0.2636 | Regression_pt: 1.7878 | Regression_sin_phi: 0.5272 | LR: 1.75e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3981s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0666 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1306 | max_tensorboard_logging: 0.3529 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5701.7 | uss_MB: 5225.3 | child_rss_MB: 3191.8 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:10,886] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:21,763] INFO : Step 40800/50000 rank0 | Train Loss: 8.1880 | Train Loss Unweighted: 2.7539 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4685 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1194 | Regression_eta: 0.0103 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.4685 | Regression_cos_phi: 0.4788 | Regression_energy: 1.6078 | Regression_eta: 0.4314 | Regression_pt: 1.6953 | Regression_sin_phi: 0.7979 | LR: 1.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3590s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0205 | max_loss: 0.0036 | max_backward: 0.0329 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3515 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5701.7 | uss_MB: 5225.5 | child_rss_MB: 3192.2 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:21,973] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:32,897] INFO : Step 40900/50000 rank0 | Train Loss: 8.1636 | Train Loss Unweighted: 2.7359 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4422 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1271 | Regression_eta: 0.0110 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6056 | Classification_binary: 2.4422 | Regression_cos_phi: 0.4703 | Regression_energy: 1.7117 | Regression_eta: 0.4625 | Regression_pt: 1.7545 | Regression_sin_phi: 0.7168 | LR: 1.72e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0418 | max_forward: 0.0159 | max_loss: 0.0032 | max_backward: 0.0336 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2738 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5701.7 | uss_MB: 5225.5 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:03:33,112] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:43,980] INFO : Step 41000/50000 rank0 | Train Loss: 8.3519 | Train Loss Unweighted: 2.8168 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.5064 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1321 | Regression_eta: 0.0095 | Regression_pt: 0.1343 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6283 | Classification_binary: 2.5064 | Regression_cos_phi: 0.4300 | Regression_energy: 1.7779 | Regression_eta: 0.3984 | Regression_pt: 1.9644 | Regression_sin_phi: 0.6464 | LR: 1.70e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4155s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0741 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1316 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5701.0 | uss_MB: 5224.2 | child_rss_MB: 3192.8 | allocated_MB: 207.0 | reserved_MB: 2760.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:03:44,190] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:44,239] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 13:03:44,321] INFO : Running validation on rank0 +rank0 [2026-10-28 13:03:44,321] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:03:44,321] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:03:45,405] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 13:03:45,550] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:03:45,683] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:03:45,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:03:45,828] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:03:45,896] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:03:45,966] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:03:46,037] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:03:46,151] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:03:46,276] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:03:46,399] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:03:46,472] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:03:46,542] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:03:46,610] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:03:46,681] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:03:46,751] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:03:46,851] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:03:46,857] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:03:46,860] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:03:46,863] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:03:46,863] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:03:46,866] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:03:46,869] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:03:46,870] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:03:46,872] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:03:46,875] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:03:46,876] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:03:46,878] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:03:46,881] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:03:46,882] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:03:46,884] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:03:46,887] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:03:52,899] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.3519 | Valid Loss=8.2598 | Train Loss Unweighted=2.8168 | Valid Loss Unweighted=2.7676 | Stale=0 | ETA=24.4m +rank0 [2026-10-28 13:03:52,908] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:52,968] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:03:52,971] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:03:53,156] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:54,543] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:55,117] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:55,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:56,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:03:56,393] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:03:56,397] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:03:56,453] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:03:56,455] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:03:56,574] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:03:58,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:03:58,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:03:59,340] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:03:59,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:04:00,115] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:04:00,119] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:04:00,173] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:04:00,176] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:04:00,295] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:04:01,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:04:02,127] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:04:02,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:04:03,230] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:04:03,440] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:04:03,447] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:04:03,700] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:04:04,158] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:05,608] INFO : Plotted jets +rank0 [2026-10-28 13:04:06,721] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:06,752] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:06,977] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:04:07,357] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:08,753] INFO : Plotted jets +rank0 [2026-10-28 13:04:10,438] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:10,470] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 13:04:10,671] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 13:04:11,089] INFO : Plotted number of elements +rank0 [2026-10-28 13:04:12,489] INFO : Plotted jets +rank0 [2026-10-28 13:04:13,592] INFO : Plotted jet ratio +rank0 [2026-10-28 13:04:24,946] INFO : Step 41100/50000 rank0 | Train Loss: 8.7194 | Train Loss Unweighted: 2.7891 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4676 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1361 | Regression_eta: 0.0102 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6856 | Classification_binary: 2.4676 | Regression_cos_phi: 0.5980 | Regression_energy: 1.8327 | Regression_eta: 0.4270 | Regression_pt: 1.9702 | Regression_sin_phi: 0.7382 | LR: 1.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4075s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0220 | max_backward: 0.0330 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3751 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5893.2 | uss_MB: 5417.4 | child_rss_MB: 3189.8 | allocated_MB: 207.0 | reserved_MB: 2618.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:04:25,159] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:36,164] INFO : Step 41200/50000 rank0 | Train Loss: 8.5416 | Train Loss Unweighted: 2.8220 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5120 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1327 | Regression_eta: 0.0116 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6543 | Classification_binary: 2.5120 | Regression_cos_phi: 0.5100 | Regression_energy: 1.7864 | Regression_eta: 0.4866 | Regression_pt: 1.8714 | Regression_sin_phi: 0.7210 | LR: 1.67e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4015s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0589 | max_loss: 0.0032 | max_backward: 0.0338 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.3371 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 5893.2 | uss_MB: 5417.5 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:04:36,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:47,131] INFO : Step 41300/50000 rank0 | Train Loss: 7.3110 | Train Loss Unweighted: 2.5589 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2925 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1133 | Regression_eta: 0.0068 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6054 | Classification_binary: 2.2925 | Regression_cos_phi: 0.3877 | Regression_energy: 1.5254 | Regression_eta: 0.2836 | Regression_pt: 1.6741 | Regression_sin_phi: 0.5423 | LR: 1.66e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4380s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0187 | max_loss: 0.0036 | max_backward: 0.0391 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.2771 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 5893.2 | uss_MB: 5418.3 | child_rss_MB: 3186.2 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:04:47,343] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:58,284] INFO : Step 41400/50000 rank0 | Train Loss: 8.4860 | Train Loss Unweighted: 2.8295 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5230 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1280 | Regression_eta: 0.0109 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7021 | Classification_binary: 2.5230 | Regression_cos_phi: 0.4525 | Regression_energy: 1.7238 | Regression_eta: 0.4569 | Regression_pt: 1.8903 | Regression_sin_phi: 0.7373 | LR: 1.64e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3708s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0156 | max_backward: 0.0334 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.3641 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5775.7 | uss_MB: 5299.4 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:04:58,496] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:09,439] INFO : Step 41500/50000 rank0 | Train Loss: 8.3682 | Train Loss Unweighted: 2.7248 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.4097 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1350 | Regression_eta: 0.0097 | Regression_pt: 0.1331 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7432 | Classification_binary: 2.4097 | Regression_cos_phi: 0.3864 | Regression_energy: 1.8170 | Regression_eta: 0.4067 | Regression_pt: 1.9472 | Regression_sin_phi: 0.6581 | LR: 1.63e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4174s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0180 | max_loss: 0.0103 | max_backward: 0.0389 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5775.7 | uss_MB: 5301.9 | child_rss_MB: 3187.8 | allocated_MB: 210.7 | reserved_MB: 2738.0 | inactive_split_MB: 25.3 +rank0 [2026-10-28 13:05:09,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:20,641] INFO : Step 41600/50000 rank0 | Train Loss: 8.6109 | Train Loss Unweighted: 2.7311 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4089 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1363 | Regression_eta: 0.0140 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6801 | Classification_binary: 2.4089 | Regression_cos_phi: 0.5013 | Regression_energy: 1.8353 | Regression_eta: 0.5888 | Regression_pt: 1.9690 | Regression_sin_phi: 0.6275 | LR: 1.61e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4528s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1213 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1318 | max_tensorboard_logging: 0.3426 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3186.3 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:20,851] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:31,816] INFO : Step 41700/50000 rank0 | Train Loss: 8.3995 | Train Loss Unweighted: 2.7988 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4845 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1362 | Regression_eta: 0.0079 | Regression_pt: 0.1349 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6028 | Classification_binary: 2.4845 | Regression_cos_phi: 0.4733 | Regression_energy: 1.8337 | Regression_eta: 0.3308 | Regression_pt: 1.9726 | Regression_sin_phi: 0.7018 | LR: 1.60e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0923 | max_forward: 0.0157 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1311 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3186.2 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:05:32,030] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:42,828] INFO : Step 41800/50000 rank0 | Train Loss: 8.4166 | Train Loss Unweighted: 2.7592 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.4485 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1312 | Regression_eta: 0.0094 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7081 | Classification_binary: 2.4485 | Regression_cos_phi: 0.4951 | Regression_energy: 1.7665 | Regression_eta: 0.3947 | Regression_pt: 1.9233 | Regression_sin_phi: 0.6803 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4545s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0159 | max_loss: 0.0816 | max_backward: 0.0338 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5776.7 | uss_MB: 5300.2 | child_rss_MB: 3185.9 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:05:43,039] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:54,032] INFO : Step 41900/50000 rank0 | Train Loss: 8.4648 | Train Loss Unweighted: 2.7754 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4581 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1364 | Regression_eta: 0.0110 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6319 | Classification_binary: 2.4581 | Regression_cos_phi: 0.4534 | Regression_energy: 1.8369 | Regression_eta: 0.4629 | Regression_pt: 1.9686 | Regression_sin_phi: 0.6529 | LR: 1.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4580s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1195 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1335 | max_tensorboard_logging: 0.3323 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5776.7 | uss_MB: 5300.4 | child_rss_MB: 3190.0 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:05:54,245] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:05,044] INFO : Step 42000/50000 rank0 | Train Loss: 8.3631 | Train Loss Unweighted: 2.8409 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.5203 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1404 | Regression_eta: 0.0072 | Regression_pt: 0.1396 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6566 | Classification_binary: 2.5203 | Regression_cos_phi: 0.3605 | Regression_energy: 1.8904 | Regression_eta: 0.3015 | Regression_pt: 2.0414 | Regression_sin_phi: 0.5925 | LR: 1.56e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0149 | max_loss: 0.1072 | max_backward: 0.0332 | max_optimizer_step: 0.1330 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5775.9 | uss_MB: 5299.2 | child_rss_MB: 3191.1 | allocated_MB: 207.0 | reserved_MB: 2738.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:06:05,256] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:05,273] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 13:06:05,457] INFO : Running validation on rank0 +rank0 [2026-10-28 13:06:05,457] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:06:05,457] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:06:06,544] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 13:06:06,689] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:06:06,823] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:06:06,898] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:06:06,967] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:06:07,036] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:06:07,106] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:06:07,176] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:06:07,289] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:06:07,422] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:06:07,555] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:06:07,628] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:06:07,699] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:06:07,768] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:06:07,838] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:06:07,907] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:06:08,006] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:06:08,012] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:06:08,015] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:06:08,017] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:06:08,020] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:06:08,021] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:06:08,024] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:06:08,025] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:06:08,027] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:06:08,030] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:06:08,031] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:06:08,033] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:06:08,036] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:06:08,038] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:06:08,039] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:06:08,042] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:06:14,093] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.3631 | Valid Loss=8.1967 | Train Loss Unweighted=2.8409 | Valid Loss Unweighted=2.7411 | Stale=0 | ETA=21.7m +rank0 [2026-10-28 13:06:14,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:14,162] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:06:14,165] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:14,344] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:15,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:16,453] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:17,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:17,580] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:17,782] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:06:17,786] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:17,840] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:06:17,843] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:17,965] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:19,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:19,876] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:20,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:21,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:21,252] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:06:21,255] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:06:21,310] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:06:21,313] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:06:21,433] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:06:22,936] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:06:23,454] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:06:23,968] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:06:24,541] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:06:24,701] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:06:24,708] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:24,959] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:06:25,411] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:26,863] INFO : Plotted jets +rank0 [2026-10-28 13:06:27,974] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:28,007] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:28,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:06:28,613] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:30,006] INFO : Plotted jets +rank0 [2026-10-28 13:06:31,640] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:31,673] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 13:06:31,873] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 13:06:32,291] INFO : Plotted number of elements +rank0 [2026-10-28 13:06:33,693] INFO : Plotted jets +rank0 [2026-10-28 13:06:34,806] INFO : Plotted jet ratio +rank0 [2026-10-28 13:06:46,208] INFO : Step 42100/50000 rank0 | Train Loss: 8.6959 | Train Loss Unweighted: 2.7661 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4464 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1345 | Regression_eta: 0.0125 | Regression_pt: 0.1330 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6923 | Classification_binary: 2.4464 | Regression_cos_phi: 0.5347 | Regression_energy: 1.8104 | Regression_eta: 0.5236 | Regression_pt: 1.9458 | Regression_sin_phi: 0.7427 | LR: 1.54e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.3746s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0594 | max_loss: 0.0033 | max_backward: 0.0337 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3199 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5967.1 | uss_MB: 5492.4 | child_rss_MB: 3193.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:06:46,421] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 1% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:06:57,321] INFO : Step 42200/50000 rank0 | Train Loss: 7.9474 | Train Loss Unweighted: 2.6970 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1265 | Regression_eta: 0.0092 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5913 | Classification_binary: 2.4026 | Regression_cos_phi: 0.3757 | Regression_energy: 1.7025 | Regression_eta: 0.3853 | Regression_pt: 1.8473 | Regression_sin_phi: 0.6428 | LR: 1.53e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4353s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1199 | max_loss: 0.0033 | max_backward: 0.0334 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3218 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5967.1 | uss_MB: 5492.4 | child_rss_MB: 3191.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:06:57,531] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 13:06:59,186] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 13:06:59,186] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:07:08,821] INFO : Step 42300/50000 rank0 | Train Loss: 7.8425 | Train Loss Unweighted: 2.6964 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4197 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1171 | Regression_eta: 0.0115 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6194 | Classification_binary: 2.4197 | Regression_cos_phi: 0.4554 | Regression_energy: 1.5768 | Regression_eta: 0.4838 | Regression_pt: 1.6603 | Regression_sin_phi: 0.6271 | LR: 1.52e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3842s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0033 | max_backward: 0.1181 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6006.0 | uss_MB: 5530.2 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:07:09,032] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:19,812] INFO : Step 42400/50000 rank0 | Train Loss: 7.8425 | Train Loss Unweighted: 2.7143 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4380 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1148 | Regression_eta: 0.0088 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6906 | Classification_binary: 2.4380 | Regression_cos_phi: 0.4260 | Regression_energy: 1.5460 | Regression_eta: 0.3680 | Regression_pt: 1.6962 | Regression_sin_phi: 0.6777 | LR: 1.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.2950s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1590 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 5847.5 | uss_MB: 5371.7 | child_rss_MB: 3192.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:07:20,040] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:31,071] INFO : Step 42500/50000 rank0 | Train Loss: 7.8462 | Train Loss Unweighted: 2.6829 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3982 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1221 | Regression_eta: 0.0088 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6587 | Classification_binary: 2.3982 | Regression_cos_phi: 0.4638 | Regression_energy: 1.6431 | Regression_eta: 0.3697 | Regression_pt: 1.7349 | Regression_sin_phi: 0.5778 | LR: 1.49e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3794s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0150 | max_loss: 0.0037 | max_backward: 0.1281 | max_optimizer_step: 0.1342 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5847.5 | uss_MB: 5372.2 | child_rss_MB: 3219.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:07:31,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:42,056] INFO : Step 42600/50000 rank0 | Train Loss: 7.4513 | Train Loss Unweighted: 2.6006 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3437 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1069 | Regression_eta: 0.0086 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5946 | Classification_binary: 2.3437 | Regression_cos_phi: 0.4742 | Regression_energy: 1.4386 | Regression_eta: 0.3614 | Regression_pt: 1.5586 | Regression_sin_phi: 0.6802 | LR: 1.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3134s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0144 | max_loss: 0.0031 | max_backward: 0.0329 | max_optimizer_step: 0.1700 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5848.5 | uss_MB: 5372.5 | child_rss_MB: 3190.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:07:42,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:53,318] INFO : Step 42700/50000 rank0 | Train Loss: 8.1799 | Train Loss Unweighted: 2.7325 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4267 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1314 | Regression_eta: 0.0098 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6217 | Classification_binary: 2.4267 | Regression_cos_phi: 0.4349 | Regression_energy: 1.7694 | Regression_eta: 0.4116 | Regression_pt: 1.9103 | Regression_sin_phi: 0.6053 | LR: 1.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.2931s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0031 | max_backward: 0.0420 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2732 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5848.5 | uss_MB: 5372.7 | child_rss_MB: 3217.1 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:07:53,530] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:04,291] INFO : Step 42800/50000 rank0 | Train Loss: 7.5789 | Train Loss Unweighted: 2.6883 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4137 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1176 | Regression_eta: 0.0083 | Regression_pt: 0.1167 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6291 | Classification_binary: 2.4137 | Regression_cos_phi: 0.3659 | Regression_energy: 1.5832 | Regression_eta: 0.3492 | Regression_pt: 1.7073 | Regression_sin_phi: 0.5306 | LR: 1.45e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3760s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0146 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.2097 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5848.5 | uss_MB: 5372.7 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:08:04,510] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 83.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:15,160] INFO : Step 42900/50000 rank0 | Train Loss: 8.1238 | Train Loss Unweighted: 2.7166 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.4233 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1243 | Regression_eta: 0.0087 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6055 | Classification_binary: 2.4233 | Regression_cos_phi: 0.4509 | Regression_energy: 1.6728 | Regression_eta: 0.3629 | Regression_pt: 1.8162 | Regression_sin_phi: 0.7922 | LR: 1.44e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3827s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0269 | max_loss: 0.0034 | max_backward: 0.0377 | max_optimizer_step: 0.1339 | max_tensorboard_logging: 0.3330 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5849.5 | uss_MB: 5375.4 | child_rss_MB: 3187.9 | allocated_MB: 210.7 | reserved_MB: 2790.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:08:15,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:26,277] INFO : Step 43000/50000 rank0 | Train Loss: 8.3864 | Train Loss Unweighted: 2.7329 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4176 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1354 | Regression_eta: 0.0106 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6885 | Classification_binary: 2.4176 | Regression_cos_phi: 0.4762 | Regression_energy: 1.8234 | Regression_eta: 0.4463 | Regression_pt: 1.9408 | Regression_sin_phi: 0.5937 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4276s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0749 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1332 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5847.1 | uss_MB: 5371.3 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 13:08:26,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:08:26,526] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 13:08:26,608] INFO : Running validation on rank0 +rank0 [2026-10-28 13:08:26,608] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:08:26,608] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:08:27,687] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 13:08:27,831] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:08:27,962] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:08:28,037] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:08:28,106] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:08:28,173] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:08:28,242] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:08:28,312] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:08:28,425] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:08:28,559] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:08:28,690] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:08:28,762] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:08:28,832] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:08:28,900] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:08:28,970] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:08:29,039] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:08:29,138] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:08:29,144] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:08:29,147] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:08:29,150] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:08:29,151] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:08:29,153] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:08:29,156] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:08:29,157] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:08:29,159] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:08:29,162] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:08:29,163] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:08:29,165] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:08:29,168] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:08:29,171] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:08:29,172] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:08:29,174] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:08:35,201] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.3864 | Valid Loss=8.2576 | Train Loss Unweighted=2.7329 | Valid Loss Unweighted=2.7510 | Stale=1 | ETA=18.9m +rank0 [2026-10-28 13:08:35,210] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:35,271] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:08:35,274] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:35,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:37,030] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:37,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:38,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:38,835] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:39,033] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:39,037] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:39,094] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:08:39,097] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:39,213] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:40,751] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:41,301] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:41,877] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:42,417] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:42,638] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:42,641] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:42,695] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:08:42,698] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:08:42,821] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:44,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:44,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:45,286] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:45,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:46,011] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:08:46,019] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:46,269] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:46,728] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:48,174] INFO : Plotted jets +rank0 [2026-10-28 13:08:49,278] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:49,310] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:49,538] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:49,920] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:51,895] INFO : Plotted jets +rank0 [2026-10-28 13:08:52,965] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:52,996] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:53,199] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:53,618] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:55,022] INFO : Plotted jets +rank0 [2026-10-28 13:08:56,122] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:07,383] INFO : Step 43100/50000 rank0 | Train Loss: 8.3640 | Train Loss Unweighted: 2.7027 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.3862 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1340 | Regression_eta: 0.0113 | Regression_pt: 0.1352 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6996 | Classification_binary: 2.3862 | Regression_cos_phi: 0.4320 | Regression_energy: 1.8043 | Regression_eta: 0.4744 | Regression_pt: 1.9767 | Regression_sin_phi: 0.5909 | LR: 1.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3758s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0186 | max_loss: 0.0035 | max_backward: 0.0341 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3244 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:07,594] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:18,558] INFO : Step 43200/50000 rank0 | Train Loss: 8.4272 | Train Loss Unweighted: 2.7483 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4232 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1415 | Regression_eta: 0.0080 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6907 | Classification_binary: 2.4232 | Regression_cos_phi: 0.4351 | Regression_energy: 1.9049 | Regression_eta: 0.3352 | Regression_pt: 2.0443 | Regression_sin_phi: 0.5938 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4048s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0747 | max_forward: 0.0155 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3189.3 | allocated_MB: 207.0 | reserved_MB: 2694.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:09:18,775] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:29,635] INFO : Step 43300/50000 rank0 | Train Loss: 8.0953 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3966 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1280 | Regression_eta: 0.0085 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6572 | Classification_binary: 2.3966 | Regression_cos_phi: 0.3799 | Regression_energy: 1.7231 | Regression_eta: 0.3580 | Regression_pt: 1.9229 | Regression_sin_phi: 0.6577 | LR: 1.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0961 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0340 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6038.9 | uss_MB: 5563.9 | child_rss_MB: 3187.9 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:09:29,850] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:40,724] INFO : Step 43400/50000 rank0 | Train Loss: 8.0748 | Train Loss Unweighted: 2.7226 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4308 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1233 | Regression_eta: 0.0085 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6795 | Classification_binary: 2.4308 | Regression_cos_phi: 0.4417 | Regression_energy: 1.6600 | Regression_eta: 0.3568 | Regression_pt: 1.7978 | Regression_sin_phi: 0.7083 | LR: 1.38e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0140 | max_loss: 0.0032 | max_backward: 0.0328 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5923.5 | uss_MB: 5447.6 | child_rss_MB: 3189.4 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:09:40,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 130.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:51,773] INFO : Step 43500/50000 rank0 | Train Loss: 7.6929 | Train Loss Unweighted: 2.7118 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4328 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1186 | Regression_eta: 0.0082 | Regression_pt: 0.1189 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6498 | Classification_binary: 2.4328 | Regression_cos_phi: 0.4191 | Regression_energy: 1.5967 | Regression_eta: 0.3448 | Regression_pt: 1.7391 | Regression_sin_phi: 0.5107 | LR: 1.37e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3974s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0409 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.3319 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5923.5 | uss_MB: 5448.6 | child_rss_MB: 3185.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:09:51,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:02,908] INFO : Step 43600/50000 rank0 | Train Loss: 7.7739 | Train Loss Unweighted: 2.6227 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3469 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1143 | Regression_eta: 0.0101 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6079 | Classification_binary: 2.3469 | Regression_cos_phi: 0.5004 | Regression_energy: 1.5386 | Regression_eta: 0.4242 | Regression_pt: 1.6946 | Regression_sin_phi: 0.6612 | LR: 1.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3766s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5923.5 | uss_MB: 5448.7 | child_rss_MB: 3183.3 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:10:03,122] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:14,059] INFO : Step 43700/50000 rank0 | Train Loss: 8.3174 | Train Loss Unweighted: 2.7847 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4843 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1267 | Regression_eta: 0.0108 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6435 | Classification_binary: 2.4843 | Regression_cos_phi: 0.4514 | Regression_energy: 1.7051 | Regression_eta: 0.4512 | Regression_pt: 1.8501 | Regression_sin_phi: 0.7318 | LR: 1.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3770s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0637 | max_forward: 0.0164 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5923.5 | uss_MB: 5448.7 | child_rss_MB: 3185.0 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:14,272] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:25,081] INFO : Step 43800/50000 rank0 | Train Loss: 7.3242 | Train Loss Unweighted: 2.5683 | Train Loss Components: Classification: 0.0156 | Classification_binary: 2.3025 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1124 | Regression_eta: 0.0078 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5305 | Classification_binary: 2.3025 | Regression_cos_phi: 0.4403 | Regression_energy: 1.5132 | Regression_eta: 0.3259 | Regression_pt: 1.6859 | Regression_sin_phi: 0.5259 | LR: 1.34e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3573s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0142 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1361 | max_tensorboard_logging: 0.3288 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5923.5 | uss_MB: 5449.2 | child_rss_MB: 3185.7 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:25,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:36,249] INFO : Step 43900/50000 rank0 | Train Loss: 8.8461 | Train Loss Unweighted: 2.8260 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5003 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1383 | Regression_eta: 0.0100 | Regression_pt: 0.1373 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6465 | Classification_binary: 2.5003 | Regression_cos_phi: 0.5751 | Regression_energy: 1.8621 | Regression_eta: 0.4175 | Regression_pt: 2.0076 | Regression_sin_phi: 0.8370 | LR: 1.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3687s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0558 | max_forward: 0.0169 | max_loss: 0.0034 | max_backward: 0.0340 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5923.5 | uss_MB: 5449.2 | child_rss_MB: 3185.6 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:10:36,460] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:47,361] INFO : Step 44000/50000 rank0 | Train Loss: 8.0328 | Train Loss Unweighted: 2.7417 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4506 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1224 | Regression_eta: 0.0104 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6476 | Classification_binary: 2.4506 | Regression_cos_phi: 0.4571 | Regression_energy: 1.6477 | Regression_eta: 0.4367 | Regression_pt: 1.8040 | Regression_sin_phi: 0.5891 | LR: 1.32e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3945s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0613 | max_forward: 0.0138 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1363 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5922.6 | uss_MB: 5447.8 | child_rss_MB: 3183.5 | allocated_MB: 207.0 | reserved_MB: 2790.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:10:47,577] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 41.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:47,595] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 13:10:47,677] INFO : Running validation on rank0 +rank0 [2026-10-28 13:10:47,677] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:10:47,677] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:10:48,740] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 13:10:48,886] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:10:49,017] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:10:49,091] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:10:49,160] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:10:49,227] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:10:49,296] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:10:49,365] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:10:49,478] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:10:49,611] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:10:49,744] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:10:49,814] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:10:49,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:10:49,952] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:10:50,021] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:10:50,090] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:10:50,189] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:10:50,195] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:10:50,197] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:10:50,200] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:10:50,203] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:10:50,204] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:10:50,206] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:10:50,207] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:10:50,210] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:10:50,213] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:10:50,215] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:10:50,218] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:10:50,220] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:10:50,223] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:10:50,224] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:10:50,226] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:10:55,962] INFO : VALIDATION | Step=44000/50000 | Train Loss=8.0328 | Valid Loss=8.1542 | Train Loss Unweighted=2.7417 | Valid Loss Unweighted=2.7294 | Stale=0 | ETA=16.1m +rank0 [2026-10-28 13:10:55,971] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:10:56,033] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:10:56,036] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:10:56,215] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:10:57,583] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:10:58,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:10:58,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:10:59,325] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:10:59,530] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:10:59,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:10:59,590] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:10:59,593] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:10:59,713] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:01,066] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:01,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:02,152] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:02,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:03,002] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:11:03,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:11:03,059] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:11:03,062] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:11:03,181] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:11:04,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:11:05,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:11:05,766] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:11:06,271] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:11:06,492] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:11:06,500] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:11:06,751] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:11:07,210] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:08,650] INFO : Plotted jets +rank0 [2026-10-28 13:11:09,757] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:09,788] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:10,013] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:11:10,395] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:11,780] INFO : Plotted jets +rank0 [2026-10-28 13:11:13,425] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:13,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 13:11:13,660] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 13:11:14,078] INFO : Plotted number of elements +rank0 [2026-10-28 13:11:15,481] INFO : Plotted jets +rank0 [2026-10-28 13:11:16,595] INFO : Plotted jet ratio +rank0 [2026-10-28 13:11:27,951] INFO : Step 44100/50000 rank0 | Train Loss: 8.2172 | Train Loss Unweighted: 2.7510 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4524 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1259 | Regression_eta: 0.0087 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6657 | Classification_binary: 2.4524 | Regression_cos_phi: 0.4852 | Regression_energy: 1.6955 | Regression_eta: 0.3641 | Regression_pt: 1.8499 | Regression_sin_phi: 0.7044 | LR: 1.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0337 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.3714 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6112.1 | uss_MB: 5637.1 | child_rss_MB: 3181.7 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:28,163] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:39,146] INFO : Step 44200/50000 rank0 | Train Loss: 7.5128 | Train Loss Unweighted: 2.6238 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3650 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1088 | Regression_eta: 0.0105 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6051 | Classification_binary: 2.3650 | Regression_cos_phi: 0.4321 | Regression_energy: 1.4643 | Regression_eta: 0.4421 | Regression_pt: 1.5398 | Regression_sin_phi: 0.6644 | LR: 1.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4559s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1138 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1371 | max_tensorboard_logging: 0.3278 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6112.1 | uss_MB: 5637.1 | child_rss_MB: 3181.9 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:11:39,359] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:50,248] INFO : Step 44300/50000 rank0 | Train Loss: 8.6279 | Train Loss Unweighted: 2.7977 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4829 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1336 | Regression_eta: 0.0122 | Regression_pt: 0.1307 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6582 | Classification_binary: 2.4829 | Regression_cos_phi: 0.5281 | Regression_energy: 1.7980 | Regression_eta: 0.5097 | Regression_pt: 1.9121 | Regression_sin_phi: 0.7389 | LR: 1.29e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4382s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0956 | max_loss: 0.0034 | max_backward: 0.0331 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3044 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6112.1 | uss_MB: 5637.2 | child_rss_MB: 3184.2 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:11:50,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:01,399] INFO : Step 44400/50000 rank0 | Train Loss: 8.0954 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4192 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1263 | Regression_eta: 0.0102 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6713 | Classification_binary: 2.4192 | Regression_cos_phi: 0.3967 | Regression_energy: 1.6997 | Regression_eta: 0.4257 | Regression_pt: 1.7697 | Regression_sin_phi: 0.7131 | LR: 1.28e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4017s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.0333 | max_loss: 0.0035 | max_backward: 0.0378 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3355 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5995.1 | uss_MB: 5522.0 | child_rss_MB: 3188.4 | allocated_MB: 210.7 | reserved_MB: 2728.0 | inactive_split_MB: 31.3 +rank0 [2026-10-28 13:12:01,612] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:12,473] INFO : Step 44500/50000 rank0 | Train Loss: 7.3138 | Train Loss Unweighted: 2.6478 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3807 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1151 | Regression_eta: 0.0074 | Regression_pt: 0.1149 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6076 | Classification_binary: 2.3807 | Regression_cos_phi: 0.3349 | Regression_energy: 1.5495 | Regression_eta: 0.3119 | Regression_pt: 1.6804 | Regression_sin_phi: 0.4488 | LR: 1.27e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4309s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1053 | max_loss: 0.0034 | max_backward: 0.0334 | max_optimizer_step: 0.1378 | max_tensorboard_logging: 0.3112 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 5996.1 | uss_MB: 5520.9 | child_rss_MB: 3185.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:12,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:23,595] INFO : Step 44600/50000 rank0 | Train Loss: 7.7606 | Train Loss Unweighted: 2.6420 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3557 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1220 | Regression_eta: 0.0072 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6208 | Classification_binary: 2.3557 | Regression_cos_phi: 0.3942 | Regression_energy: 1.6420 | Regression_eta: 0.3007 | Regression_pt: 1.8076 | Regression_sin_phi: 0.6395 | LR: 1.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3748s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0455 | max_loss: 0.0035 | max_backward: 0.0333 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3290 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5996.1 | uss_MB: 5521.0 | child_rss_MB: 3184.1 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:23,809] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:34,790] INFO : Step 44700/50000 rank0 | Train Loss: 8.2437 | Train Loss Unweighted: 2.6901 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3871 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1284 | Regression_eta: 0.0101 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6234 | Classification_binary: 2.3871 | Regression_cos_phi: 0.5041 | Regression_energy: 1.7289 | Regression_eta: 0.4234 | Regression_pt: 1.8710 | Regression_sin_phi: 0.7059 | LR: 1.25e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4121s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0838 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5996.1 | uss_MB: 5521.4 | child_rss_MB: 3184.0 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:35,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:45,818] INFO : Step 44800/50000 rank0 | Train Loss: 7.7908 | Train Loss Unweighted: 2.7207 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1173 | Regression_eta: 0.0098 | Regression_pt: 0.1150 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6200 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4608 | Regression_energy: 1.5791 | Regression_eta: 0.4097 | Regression_pt: 1.6816 | Regression_sin_phi: 0.5952 | LR: 1.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0906 | max_loss: 0.0034 | max_backward: 0.0326 | max_optimizer_step: 0.1367 | max_tensorboard_logging: 0.3180 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 5996.1 | uss_MB: 5521.4 | child_rss_MB: 3186.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:12:46,029] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:56,957] INFO : Step 44900/50000 rank0 | Train Loss: 8.0838 | Train Loss Unweighted: 2.7536 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.4631 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1237 | Regression_eta: 0.0113 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6161 | Classification_binary: 2.4631 | Regression_cos_phi: 0.4318 | Regression_energy: 1.6656 | Regression_eta: 0.4719 | Regression_pt: 1.7703 | Regression_sin_phi: 0.6650 | LR: 1.23e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3849s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0310 | max_loss: 0.0035 | max_backward: 0.0337 | max_optimizer_step: 0.1380 | max_tensorboard_logging: 0.3194 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5997.1 | uss_MB: 5521.8 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:12:57,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:08,082] INFO : Step 45000/50000 rank0 | Train Loss: 7.9905 | Train Loss Unweighted: 2.7391 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4452 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1253 | Regression_eta: 0.0073 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5986 | Classification_binary: 2.4452 | Regression_cos_phi: 0.4390 | Regression_energy: 1.6874 | Regression_eta: 0.3046 | Regression_pt: 1.8635 | Regression_sin_phi: 0.6521 | LR: 1.22e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0918 | max_loss: 0.0035 | max_backward: 0.0334 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3298 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5997.1 | uss_MB: 5521.9 | child_rss_MB: 3185.5 | allocated_MB: 207.0 | reserved_MB: 2728.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:13:08,295] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:08,350] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 13:13:08,434] INFO : Running validation on rank0 +rank0 [2026-10-28 13:13:08,434] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:13:08,435] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:13:09,513] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 13:13:09,658] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:13:09,791] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:13:09,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:13:09,937] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:13:10,005] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:13:10,075] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:13:10,145] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:13:10,259] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:13:10,386] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:13:10,510] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:13:10,583] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:13:10,653] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:13:10,722] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:13:10,793] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:13:10,862] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:13:10,960] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:13:10,966] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:13:10,968] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:13:10,971] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:13:10,972] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:13:10,974] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:13:10,977] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:13:10,978] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:13:10,981] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:13:10,983] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:13:10,986] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:13:10,987] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:13:10,989] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:13:10,992] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:13:10,993] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:13:10,996] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:13:16,801] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.9905 | Valid Loss=8.1417 | Train Loss Unweighted=2.7391 | Valid Loss Unweighted=2.7311 | Stale=0 | ETA=13.4m +rank0 [2026-10-28 13:13:16,810] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:16,871] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:13:16,874] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:17,062] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:18,641] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:19,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:19,969] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:20,635] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:20,848] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:13:20,851] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:20,908] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:13:20,911] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:21,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:22,533] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:23,078] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:23,709] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:24,243] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:24,456] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:13:24,459] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:13:24,514] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:13:24,516] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:13:24,638] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:13:25,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:13:26,608] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:13:27,134] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:13:27,754] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:13:27,996] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:13:28,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:28,257] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:13:28,719] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:30,184] INFO : Plotted jets +rank0 [2026-10-28 13:13:31,303] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:31,335] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:31,560] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:13:31,945] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:33,355] INFO : Plotted jets +rank0 [2026-10-28 13:13:35,045] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:35,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 13:13:35,280] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 13:13:35,703] INFO : Plotted number of elements +rank0 [2026-10-28 13:13:37,122] INFO : Plotted jets +rank0 [2026-10-28 13:13:38,231] INFO : Plotted jet ratio +rank0 [2026-10-28 13:13:49,604] INFO : Step 45100/50000 rank0 | Train Loss: 7.8712 | Train Loss Unweighted: 2.7146 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4422 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1115 | Regression_eta: 0.0100 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6999 | Classification_binary: 2.4422 | Regression_cos_phi: 0.4933 | Regression_energy: 1.5006 | Regression_eta: 0.4190 | Regression_pt: 1.6500 | Regression_sin_phi: 0.6661 | LR: 1.21e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4258s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0063 | max_backward: 0.0340 | max_optimizer_step: 0.1406 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6194.6 | uss_MB: 5721.1 | child_rss_MB: 3186.1 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:13:49,818] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:00,821] INFO : Step 45200/50000 rank0 | Train Loss: 8.3033 | Train Loss Unweighted: 2.7416 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4403 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1257 | Regression_eta: 0.0102 | Regression_pt: 0.1280 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6565 | Classification_binary: 2.4403 | Regression_cos_phi: 0.4630 | Regression_energy: 1.6919 | Regression_eta: 0.4282 | Regression_pt: 1.8726 | Regression_sin_phi: 0.7509 | LR: 1.20e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0321 | max_loss: 0.0035 | max_backward: 0.0331 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.3155 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.4 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:01,036] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:12,042] INFO : Step 45300/50000 rank0 | Train Loss: 7.8314 | Train Loss Unweighted: 2.6900 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3971 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1267 | Regression_eta: 0.0082 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6001 | Classification_binary: 2.3971 | Regression_cos_phi: 0.3870 | Regression_energy: 1.7051 | Regression_eta: 0.3421 | Regression_pt: 1.8473 | Regression_sin_phi: 0.5527 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3948s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0697 | max_forward: 0.0150 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1396 | max_tensorboard_logging: 0.2713 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:12,257] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:23,135] INFO : Step 45400/50000 rank0 | Train Loss: 7.7189 | Train Loss Unweighted: 2.6456 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.3708 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1162 | Regression_eta: 0.0111 | Regression_pt: 0.1147 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5721 | Classification_binary: 2.3708 | Regression_cos_phi: 0.4330 | Regression_energy: 1.5645 | Regression_eta: 0.4657 | Regression_pt: 1.6769 | Regression_sin_phi: 0.6358 | LR: 1.19e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4895s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0016 | max_forward: 0.1087 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3103 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6194.6 | uss_MB: 5721.2 | child_rss_MB: 3183.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:14:23,351] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:34,247] INFO : Step 45500/50000 rank0 | Train Loss: 8.0477 | Train Loss Unweighted: 2.6347 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3545 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1160 | Regression_eta: 0.0108 | Regression_pt: 0.1148 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6250 | Classification_binary: 2.3545 | Regression_cos_phi: 0.5021 | Regression_energy: 1.5611 | Regression_eta: 0.4527 | Regression_pt: 1.6797 | Regression_sin_phi: 0.8727 | LR: 1.18e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3392s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.0317 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6070.6 | uss_MB: 5597.4 | child_rss_MB: 3186.9 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:34,458] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:45,389] INFO : Step 45600/50000 rank0 | Train Loss: 7.7775 | Train Loss Unweighted: 2.6670 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3969 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1120 | Regression_eta: 0.0122 | Regression_pt: 0.1103 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6476 | Classification_binary: 2.3969 | Regression_cos_phi: 0.4565 | Regression_energy: 1.5081 | Regression_eta: 0.5107 | Regression_pt: 1.6127 | Regression_sin_phi: 0.6451 | LR: 1.17e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4450s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1112 | max_loss: 0.0034 | max_backward: 0.0328 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3084 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6070.6 | uss_MB: 5597.9 | child_rss_MB: 3183.3 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:14:45,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:56,526] INFO : Step 45700/50000 rank0 | Train Loss: 8.4950 | Train Loss Unweighted: 2.8061 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4934 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1343 | Regression_eta: 0.0109 | Regression_pt: 0.1312 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6251 | Classification_binary: 2.4934 | Regression_cos_phi: 0.4800 | Regression_energy: 1.8084 | Regression_eta: 0.4587 | Regression_pt: 1.9191 | Regression_sin_phi: 0.7103 | LR: 1.16e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4092s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0714 | max_loss: 0.0034 | max_backward: 0.0335 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.3260 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6070.6 | uss_MB: 5598.2 | child_rss_MB: 3182.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:14:56,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:07,629] INFO : Step 45800/50000 rank0 | Train Loss: 7.5938 | Train Loss Unweighted: 2.6176 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3442 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1165 | Regression_eta: 0.0079 | Regression_pt: 0.1145 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6490 | Classification_binary: 2.3442 | Regression_cos_phi: 0.4050 | Regression_energy: 1.5682 | Regression_eta: 0.3304 | Regression_pt: 1.6752 | Regression_sin_phi: 0.6219 | LR: 1.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0634 | max_loss: 0.0034 | max_backward: 0.0327 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3189 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6070.6 | uss_MB: 5598.0 | child_rss_MB: 3182.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:15:07,843] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:18,755] INFO : Step 45900/50000 rank0 | Train Loss: 8.6179 | Train Loss Unweighted: 2.7080 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3983 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1288 | Regression_eta: 0.0140 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6312 | Classification_binary: 2.3983 | Regression_cos_phi: 0.5843 | Regression_energy: 1.7344 | Regression_eta: 0.5850 | Regression_pt: 1.8597 | Regression_sin_phi: 0.8250 | LR: 1.15e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4395s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0998 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1382 | max_tensorboard_logging: 0.3133 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6071.6 | uss_MB: 5598.2 | child_rss_MB: 3186.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:18,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:29,867] INFO : Step 46000/50000 rank0 | Train Loss: 8.1133 | Train Loss Unweighted: 2.7475 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4581 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1218 | Regression_eta: 0.0102 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6259 | Classification_binary: 2.4581 | Regression_cos_phi: 0.4685 | Regression_energy: 1.6400 | Regression_eta: 0.4273 | Regression_pt: 1.7720 | Regression_sin_phi: 0.7213 | LR: 1.14e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1198 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3292 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 6071.6 | uss_MB: 5598.6 | child_rss_MB: 3184.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:15:30,079] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:30,126] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 13:15:30,208] INFO : Running validation on rank0 +rank0 [2026-10-28 13:15:30,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:15:30,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:15:31,274] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 13:15:31,421] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:15:31,553] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:15:31,627] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:15:31,695] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:15:31,763] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:15:31,833] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:15:31,902] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:15:32,015] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:15:32,139] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:15:32,262] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:15:32,334] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:15:32,404] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:15:32,472] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:15:32,541] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:15:32,611] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:15:32,709] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:15:32,716] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:15:32,718] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:15:32,721] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:15:32,724] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:15:32,725] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:15:32,728] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:15:32,730] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:15:32,731] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:15:32,734] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:15:32,737] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:15:32,737] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:15:32,740] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:15:32,743] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:15:32,743] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:15:32,746] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:15:38,472] INFO : VALIDATION | Step=46000/50000 | Train Loss=8.1133 | Valid Loss=8.1662 | Train Loss Unweighted=2.7475 | Valid Loss Unweighted=2.7221 | Stale=1 | ETA=10.7m +rank0 [2026-10-28 13:15:38,480] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:38,568] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:15:38,571] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:38,784] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:40,200] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:40,779] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:41,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:41,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:42,137] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:15:42,140] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:42,197] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:15:42,199] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:42,322] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:43,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:44,474] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:45,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:45,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:45,799] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:15:45,803] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:45,857] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:15:45,860] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:15:45,991] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:47,505] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:48,123] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:48,649] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:49,254] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:49,474] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:15:49,482] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:49,734] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:50,199] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:51,656] INFO : Plotted jets +rank0 [2026-10-28 13:15:52,774] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:52,807] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:53,033] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:53,523] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:54,924] INFO : Plotted jets +rank0 [2026-10-28 13:15:56,587] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:56,619] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:56,826] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:57,249] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:58,663] INFO : Plotted jets +rank0 [2026-10-28 13:15:59,782] INFO : Plotted jet ratio +rank0 [2026-10-28 13:16:11,189] INFO : Step 46100/50000 rank0 | Train Loss: 7.9889 | Train Loss Unweighted: 2.6806 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3861 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1256 | Regression_eta: 0.0089 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6019 | Classification_binary: 2.3861 | Regression_cos_phi: 0.4240 | Regression_energy: 1.6907 | Regression_eta: 0.3711 | Regression_pt: 1.8441 | Regression_sin_phi: 0.6710 | LR: 1.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4505s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0159 | max_loss: 0.0164 | max_backward: 0.0335 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3765 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6270.8 | uss_MB: 5796.7 | child_rss_MB: 3185.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:11,400] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:22,379] INFO : Step 46200/50000 rank0 | Train Loss: 8.3701 | Train Loss Unweighted: 2.7187 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4015 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1382 | Regression_eta: 0.0087 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6451 | Classification_binary: 2.4015 | Regression_cos_phi: 0.4969 | Regression_energy: 1.8611 | Regression_eta: 0.3652 | Regression_pt: 1.9601 | Regression_sin_phi: 0.6403 | LR: 1.13e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.4597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1056 | max_loss: 0.0034 | max_backward: 0.0344 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3073 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6270.8 | uss_MB: 5796.7 | child_rss_MB: 3187.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:16:22,592] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:33,479] INFO : Step 46300/50000 rank0 | Train Loss: 8.1265 | Train Loss Unweighted: 2.6285 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3179 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1329 | Regression_eta: 0.0085 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6586 | Classification_binary: 2.3179 | Regression_cos_phi: 0.5081 | Regression_energy: 1.7890 | Regression_eta: 0.3580 | Regression_pt: 1.9537 | Regression_sin_phi: 0.5413 | LR: 1.12e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4506s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1113 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1391 | max_tensorboard_logging: 0.3075 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6270.8 | uss_MB: 5796.8 | child_rss_MB: 3189.4 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 21.0 +rank0 [2026-10-28 13:16:33,694] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:44,615] INFO : Step 46400/50000 rank0 | Train Loss: 7.6373 | Train Loss Unweighted: 2.6393 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1160 | Regression_eta: 0.0087 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6182 | Classification_binary: 2.3645 | Regression_cos_phi: 0.4263 | Regression_energy: 1.5610 | Regression_eta: 0.3652 | Regression_pt: 1.7045 | Regression_sin_phi: 0.5976 | LR: 1.11e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4639s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1053 | max_loss: 0.0034 | max_backward: 0.0387 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 6270.8 | uss_MB: 5798.9 | child_rss_MB: 3189.3 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:16:44,826] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:16:55,623] INFO : Step 46500/50000 rank0 | Train Loss: 7.8443 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4147 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1181 | Regression_eta: 0.0084 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6795 | Classification_binary: 2.4147 | Regression_cos_phi: 0.4598 | Regression_energy: 1.5896 | Regression_eta: 0.3512 | Regression_pt: 1.6943 | Regression_sin_phi: 0.6552 | LR: 1.11e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0164 | max_loss: 0.0041 | max_backward: 0.0337 | max_optimizer_step: 0.1366 | max_tensorboard_logging: 0.3664 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6144.4 | uss_MB: 5669.2 | child_rss_MB: 3190.1 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:16:55,834] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:06,829] INFO : Step 46600/50000 rank0 | Train Loss: 8.0785 | Train Loss Unweighted: 2.6678 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3713 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1259 | Regression_eta: 0.0093 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6009 | Classification_binary: 2.3713 | Regression_cos_phi: 0.4793 | Regression_energy: 1.6945 | Regression_eta: 0.3878 | Regression_pt: 1.8426 | Regression_sin_phi: 0.7023 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4232s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0801 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1396 | max_tensorboard_logging: 0.3072 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6144.4 | uss_MB: 5669.4 | child_rss_MB: 3187.0 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:17:07,043] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 40.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:17,875] INFO : Step 46700/50000 rank0 | Train Loss: 8.4293 | Train Loss Unweighted: 2.6965 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3756 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1354 | Regression_eta: 0.0100 | Regression_pt: 0.1394 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6669 | Classification_binary: 2.3756 | Regression_cos_phi: 0.4410 | Regression_energy: 1.8227 | Regression_eta: 0.4202 | Regression_pt: 2.0384 | Regression_sin_phi: 0.6646 | LR: 1.10e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.4754s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0513 | max_backward: 0.0341 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6144.4 | uss_MB: 5669.8 | child_rss_MB: 3188.7 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:18,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:28,987] INFO : Step 46800/50000 rank0 | Train Loss: 8.0714 | Train Loss Unweighted: 2.6744 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3652 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1356 | Regression_eta: 0.0085 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6136 | Classification_binary: 2.3652 | Regression_cos_phi: 0.4432 | Regression_energy: 1.8257 | Regression_eta: 0.3566 | Regression_pt: 1.9327 | Regression_sin_phi: 0.5346 | LR: 1.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4253s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.0172 | max_loss: 0.0034 | max_backward: 0.0376 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.3662 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6145.4 | uss_MB: 5672.6 | child_rss_MB: 3189.3 | allocated_MB: 210.7 | reserved_MB: 2762.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:17:29,198] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:40,148] INFO : Step 46900/50000 rank0 | Train Loss: 8.1200 | Train Loss Unweighted: 2.8019 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4978 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1324 | Regression_eta: 0.0078 | Regression_pt: 0.1306 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6368 | Classification_binary: 2.4978 | Regression_cos_phi: 0.4170 | Regression_energy: 1.7820 | Regression_eta: 0.3259 | Regression_pt: 1.9098 | Regression_sin_phi: 0.5507 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4746s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1167 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.3294 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6145.4 | uss_MB: 5670.5 | child_rss_MB: 3187.8 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:40,361] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:51,355] INFO : Step 47000/50000 rank0 | Train Loss: 7.7272 | Train Loss Unweighted: 2.6824 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3947 | Regression_cos_phi: 0.0055 | Regression_energy: 0.1238 | Regression_eta: 0.0084 | Regression_pt: 0.1234 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6709 | Classification_binary: 2.3947 | Regression_cos_phi: 0.2990 | Regression_energy: 1.6664 | Regression_eta: 0.3533 | Regression_pt: 1.8049 | Regression_sin_phi: 0.5380 | LR: 1.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4217s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1035 | max_forward: 0.0154 | max_loss: 0.0034 | max_backward: 0.0342 | max_optimizer_step: 0.1397 | max_tensorboard_logging: 0.2695 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6145.4 | uss_MB: 5670.6 | child_rss_MB: 3185.6 | allocated_MB: 207.0 | reserved_MB: 2762.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:17:51,571] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:51,588] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 13:17:51,674] INFO : Running validation on rank0 +rank0 [2026-10-28 13:17:51,674] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:17:51,674] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:17:52,748] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 13:17:52,895] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:17:53,027] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:17:53,103] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:17:53,171] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:17:53,240] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:17:53,311] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:17:53,380] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:17:53,494] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:17:53,619] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:17:53,742] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:17:53,814] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:17:53,884] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:17:53,952] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:17:54,022] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:17:54,093] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:17:54,192] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:17:54,199] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:17:54,201] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:17:54,204] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:17:54,205] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:17:54,207] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:17:54,210] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:17:54,213] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:17:54,214] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:17:54,216] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:17:54,218] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:17:54,219] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:17:54,222] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:17:54,225] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:17:54,227] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:17:54,228] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:17:59,969] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.7272 | Valid Loss=8.1290 | Train Loss Unweighted=2.6824 | Valid Loss Unweighted=2.7163 | Stale=0 | ETA=8.0m +rank0 [2026-10-28 13:17:59,977] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:00,039] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:18:00,043] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:00,228] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:01,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:02,220] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:02,794] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:03,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:03,571] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:03,574] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:03,630] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:18:03,633] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:03,746] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:05,118] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:05,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:06,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:06,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:07,065] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:07,068] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:18:07,121] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:18:07,124] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:18:07,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:18:08,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:18:09,371] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:18:09,932] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:18:10,436] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:18:10,600] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:18:10,608] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:10,857] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:18:11,314] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:12,765] INFO : Plotted jets +rank0 [2026-10-28 13:18:13,871] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:13,905] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:14,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:18:14,511] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:15,907] INFO : Plotted jets +rank0 [2026-10-28 13:18:17,545] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:17,578] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 13:18:17,781] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 13:18:18,201] INFO : Plotted number of elements +rank0 [2026-10-28 13:18:19,599] INFO : Plotted jets +rank0 [2026-10-28 13:18:20,698] INFO : Plotted jet ratio +rank0 [2026-10-28 13:18:32,134] INFO : Step 47100/50000 rank0 | Train Loss: 7.9423 | Train Loss Unweighted: 2.5714 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2653 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1328 | Regression_eta: 0.0087 | Regression_pt: 0.1321 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5915 | Classification_binary: 2.2653 | Regression_cos_phi: 0.4266 | Regression_energy: 1.7876 | Regression_eta: 0.3655 | Regression_pt: 1.9319 | Regression_sin_phi: 0.5738 | LR: 1.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.2996 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6343.4 | uss_MB: 5867.4 | child_rss_MB: 3183.9 | allocated_MB: 207.0 | reserved_MB: 2706.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:32,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:43,272] INFO : Step 47200/50000 rank0 | Train Loss: 8.1329 | Train Loss Unweighted: 2.7013 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4026 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1269 | Regression_eta: 0.0097 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6403 | Classification_binary: 2.4026 | Regression_cos_phi: 0.4725 | Regression_energy: 1.7079 | Regression_eta: 0.4084 | Regression_pt: 1.8475 | Regression_sin_phi: 0.6538 | LR: 1.07e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4445s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0736 | max_loss: 0.0035 | max_backward: 0.0328 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3186.5 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:18:43,485] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:54,390] INFO : Step 47300/50000 rank0 | Train Loss: 8.0177 | Train Loss Unweighted: 2.7304 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4432 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1216 | Regression_eta: 0.0093 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6534 | Classification_binary: 2.4432 | Regression_cos_phi: 0.4185 | Regression_energy: 1.6371 | Regression_eta: 0.3884 | Regression_pt: 1.7595 | Regression_sin_phi: 0.7178 | LR: 1.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4623s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1073 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1381 | max_tensorboard_logging: 0.3172 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3185.8 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:18:54,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:05,520] INFO : Step 47400/50000 rank0 | Train Loss: 8.4397 | Train Loss Unweighted: 2.7208 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4022 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1352 | Regression_eta: 0.0096 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6499 | Classification_binary: 2.4022 | Regression_cos_phi: 0.4168 | Regression_energy: 1.8204 | Regression_eta: 0.4003 | Regression_pt: 2.0143 | Regression_sin_phi: 0.7358 | LR: 1.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4501s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0901 | max_loss: 0.0035 | max_backward: 0.0338 | max_optimizer_step: 0.1405 | max_tensorboard_logging: 0.3035 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6343.4 | uss_MB: 5867.5 | child_rss_MB: 3188.1 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:05,732] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:16,665] INFO : Step 47500/50000 rank0 | Train Loss: 8.3159 | Train Loss Unweighted: 2.7724 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4599 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1349 | Regression_eta: 0.0096 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6240 | Classification_binary: 2.4599 | Regression_cos_phi: 0.4146 | Regression_energy: 1.8158 | Regression_eta: 0.4028 | Regression_pt: 1.9595 | Regression_sin_phi: 0.6394 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.3710s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0135 | max_loss: 0.0036 | max_backward: 0.0332 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3100 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6218.1 | uss_MB: 5741.1 | child_rss_MB: 3189.2 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:16,879] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:27,715] INFO : Step 47600/50000 rank0 | Train Loss: 7.2314 | Train Loss Unweighted: 2.5732 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3173 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1084 | Regression_eta: 0.0075 | Regression_pt: 0.1079 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6020 | Classification_binary: 2.3173 | Regression_cos_phi: 0.3945 | Regression_energy: 1.4590 | Regression_eta: 0.3123 | Regression_pt: 1.5785 | Regression_sin_phi: 0.5678 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0143 | max_loss: 0.0154 | max_backward: 0.0326 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3681 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 6219.1 | uss_MB: 5742.1 | child_rss_MB: 3183.6 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:27,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:38,930] INFO : Step 47700/50000 rank0 | Train Loss: 8.7205 | Train Loss Unweighted: 2.7929 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4667 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1398 | Regression_eta: 0.0111 | Regression_pt: 0.1384 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6089 | Classification_binary: 2.4667 | Regression_cos_phi: 0.5170 | Regression_energy: 1.8824 | Regression_eta: 0.4641 | Regression_pt: 2.0236 | Regression_sin_phi: 0.7580 | LR: 1.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3574s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0160 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1387 | max_tensorboard_logging: 0.3059 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 6219.1 | uss_MB: 5742.5 | child_rss_MB: 3182.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:19:39,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:49,975] INFO : Step 47800/50000 rank0 | Train Loss: 8.1948 | Train Loss Unweighted: 2.8052 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5044 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1296 | Regression_eta: 0.0090 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6944 | Classification_binary: 2.5044 | Regression_cos_phi: 0.3961 | Regression_energy: 1.7449 | Regression_eta: 0.3762 | Regression_pt: 1.8501 | Regression_sin_phi: 0.6288 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.3980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0093 | max_backward: 0.0338 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.3546 | max_tensorboard_flush: 0.0092 | Memory: rss_MB: 6219.1 | uss_MB: 5742.5 | child_rss_MB: 3187.1 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:19:50,190] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:01,122] INFO : Step 47900/50000 rank0 | Train Loss: 8.3513 | Train Loss Unweighted: 2.7511 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4495 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1263 | Regression_eta: 0.0104 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6759 | Classification_binary: 2.4495 | Regression_cos_phi: 0.4568 | Regression_energy: 1.6998 | Regression_eta: 0.4363 | Regression_pt: 1.8551 | Regression_sin_phi: 0.7780 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4341s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0151 | max_loss: 0.0300 | max_backward: 0.0338 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3658 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6218.1 | uss_MB: 5742.5 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2802.0 | inactive_split_MB: 23.0 +rank0 [2026-10-28 13:20:01,336] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:12,300] INFO : Step 48000/50000 rank0 | Train Loss: 7.7611 | Train Loss Unweighted: 2.7114 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4356 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1167 | Regression_eta: 0.0083 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6546 | Classification_binary: 2.4356 | Regression_cos_phi: 0.4509 | Regression_energy: 1.5713 | Regression_eta: 0.3478 | Regression_pt: 1.6883 | Regression_sin_phi: 0.6126 | LR: 1.04e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4319s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1066 | max_loss: 0.0035 | max_backward: 0.0392 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6219.1 | uss_MB: 5745.4 | child_rss_MB: 3186.5 | allocated_MB: 210.7 | reserved_MB: 2802.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:20:12,512] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:12,586] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 13:20:12,669] INFO : Running validation on rank0 +rank0 [2026-10-28 13:20:12,669] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:20:12,669] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:20:13,725] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 13:20:13,870] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:20:14,002] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:20:14,076] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:20:14,145] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:20:14,213] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:20:14,282] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:20:14,351] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:20:14,464] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:20:14,597] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:20:14,729] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:20:14,801] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:20:14,870] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:20:14,938] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:20:15,007] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:20:15,077] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:20:15,176] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:20:15,182] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:20:15,184] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:20:15,185] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:20:15,188] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:20:15,189] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:20:15,192] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:20:15,194] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:20:15,195] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:20:15,198] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:20:15,201] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:20:15,203] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:20:15,204] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:20:15,213] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:20:15,224] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:20:15,227] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:20:20,936] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.7611 | Valid Loss=8.1381 | Train Loss Unweighted=2.7114 | Valid Loss Unweighted=2.7130 | Stale=1 | ETA=5.3m +rank0 [2026-10-28 13:20:20,944] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:21,006] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:20:21,010] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:21,190] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:22,607] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:23,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:23,801] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:24,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:24,520] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:20:24,523] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:24,579] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:20:24,582] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:24,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:26,225] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:26,773] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:27,332] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:27,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:28,100] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:20:28,103] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:20:28,156] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:20:28,159] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:20:28,289] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:20:29,731] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:20:30,248] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:20:30,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:20:31,369] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:20:31,616] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:20:31,623] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:31,875] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:20:32,330] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:33,770] INFO : Plotted jets +rank0 [2026-10-28 13:20:34,883] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:34,915] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:35,143] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:20:35,522] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:36,910] INFO : Plotted jets +rank0 [2026-10-28 13:20:38,598] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:38,630] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 13:20:38,831] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 13:20:39,249] INFO : Plotted number of elements +rank0 [2026-10-28 13:20:40,643] INFO : Plotted jets +rank0 [2026-10-28 13:20:41,749] INFO : Plotted jet ratio +rank0 [2026-10-28 13:20:53,111] INFO : Step 48100/50000 rank0 | Train Loss: 7.7659 | Train Loss Unweighted: 2.6934 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.4196 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1154 | Regression_eta: 0.0094 | Regression_pt: 0.1151 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5734 | Classification_binary: 2.4196 | Regression_cos_phi: 0.4520 | Regression_energy: 1.5541 | Regression_eta: 0.3943 | Regression_pt: 1.6828 | Regression_sin_phi: 0.6898 | LR: 1.03e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4563s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0046 | max_backward: 0.0343 | max_optimizer_step: 0.1428 | max_tensorboard_logging: 0.3783 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6406.7 | uss_MB: 5931.6 | child_rss_MB: 3184.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:20:53,324] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:04,656] INFO : Step 48200/50000 rank0 | Train Loss: 7.8099 | Train Loss Unweighted: 2.6632 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3723 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1259 | Regression_eta: 0.0077 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6089 | Classification_binary: 2.3723 | Regression_cos_phi: 0.3881 | Regression_energy: 1.6955 | Regression_eta: 0.3240 | Regression_pt: 1.8223 | Regression_sin_phi: 0.5987 | LR: 1.03e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0148 | max_loss: 0.0035 | max_backward: 0.1386 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6406.7 | uss_MB: 5931.6 | child_rss_MB: 3211.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:04,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 126.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:15,994] INFO : Step 48300/50000 rank0 | Train Loss: 7.5406 | Train Loss Unweighted: 2.6284 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3625 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1113 | Regression_eta: 0.0096 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5984 | Classification_binary: 2.3625 | Regression_cos_phi: 0.3951 | Regression_energy: 1.4990 | Regression_eta: 0.4022 | Regression_pt: 1.6371 | Regression_sin_phi: 0.6463 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0155 | max_loss: 0.1058 | max_backward: 0.0340 | max_optimizer_step: 0.1429 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6406.7 | uss_MB: 5931.7 | child_rss_MB: 3184.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:16,204] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 125.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:27,175] INFO : Step 48400/50000 rank0 | Train Loss: 7.8456 | Train Loss Unweighted: 2.6842 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4080 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1136 | Regression_eta: 0.0102 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7221 | Classification_binary: 2.4080 | Regression_cos_phi: 0.5257 | Regression_energy: 1.5294 | Regression_eta: 0.4255 | Regression_pt: 1.6725 | Regression_sin_phi: 0.5623 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4890s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1095 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6406.7 | uss_MB: 5931.7 | child_rss_MB: 3187.6 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 31.0 +rank0 [2026-10-28 13:21:27,394] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:38,353] INFO : Step 48500/50000 rank0 | Train Loss: 7.7940 | Train Loss Unweighted: 2.6853 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4136 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1140 | Regression_eta: 0.0082 | Regression_pt: 0.1133 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5911 | Classification_binary: 2.4136 | Regression_cos_phi: 0.5096 | Regression_energy: 1.5349 | Regression_eta: 0.3455 | Regression_pt: 1.6572 | Regression_sin_phi: 0.7420 | LR: 1.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4325s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0892 | max_forward: 0.0153 | max_loss: 0.0034 | max_backward: 0.0336 | max_optimizer_step: 0.1399 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6291.3 | uss_MB: 5815.9 | child_rss_MB: 3187.2 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:38,570] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:49,470] INFO : Step 48600/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.6888 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3816 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1264 | Regression_eta: 0.0200 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6609 | Classification_binary: 2.3816 | Regression_cos_phi: 0.4993 | Regression_energy: 1.7016 | Regression_eta: 0.8389 | Regression_pt: 1.8090 | Regression_sin_phi: 0.6690 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.3793s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0382 | max_forward: 0.0150 | max_loss: 0.0034 | max_backward: 0.0338 | max_optimizer_step: 0.1411 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6291.3 | uss_MB: 5817.0 | child_rss_MB: 3187.7 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:21:49,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 129.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:00,590] INFO : Step 48700/50000 rank0 | Train Loss: 8.3131 | Train Loss Unweighted: 2.6722 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3677 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1266 | Regression_eta: 0.0111 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6811 | Classification_binary: 2.3677 | Regression_cos_phi: 0.5424 | Regression_energy: 1.7049 | Regression_eta: 0.4662 | Regression_pt: 1.8740 | Regression_sin_phi: 0.6768 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0869 | max_forward: 0.0181 | max_loss: 0.0037 | max_backward: 0.0390 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.2772 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6291.3 | uss_MB: 5819.2 | child_rss_MB: 3187.0 | allocated_MB: 210.7 | reserved_MB: 2746.0 | inactive_split_MB: 29.3 +rank0 [2026-10-28 13:22:00,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:11,585] INFO : Step 48800/50000 rank0 | Train Loss: 7.8636 | Train Loss Unweighted: 2.5751 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.2762 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1298 | Regression_eta: 0.0068 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5841 | Classification_binary: 2.2762 | Regression_cos_phi: 0.4613 | Regression_energy: 1.7470 | Regression_eta: 0.2852 | Regression_pt: 1.8829 | Regression_sin_phi: 0.6270 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0577 | max_loss: 0.0036 | max_backward: 0.0328 | max_optimizer_step: 0.1388 | max_tensorboard_logging: 0.3223 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6291.3 | uss_MB: 5817.5 | child_rss_MB: 3186.0 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:11,800] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:22,699] INFO : Step 48900/50000 rank0 | Train Loss: 7.7267 | Train Loss Unweighted: 2.5843 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3012 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1202 | Regression_eta: 0.0093 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6047 | Classification_binary: 2.3012 | Regression_cos_phi: 0.4087 | Regression_energy: 1.6188 | Regression_eta: 0.3908 | Regression_pt: 1.7556 | Regression_sin_phi: 0.6468 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4416s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0862 | max_loss: 0.0034 | max_backward: 0.0333 | max_optimizer_step: 0.1377 | max_tensorboard_logging: 0.3240 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6291.3 | uss_MB: 5817.6 | child_rss_MB: 3187.1 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:22:22,911] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:33,731] INFO : Step 49000/50000 rank0 | Train Loss: 7.5248 | Train Loss Unweighted: 2.6023 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3189 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1233 | Regression_eta: 0.0080 | Regression_pt: 0.1223 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5841 | Classification_binary: 2.3189 | Regression_cos_phi: 0.3602 | Regression_energy: 1.6599 | Regression_eta: 0.3363 | Regression_pt: 1.7885 | Regression_sin_phi: 0.4769 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0158 | max_loss: 0.0060 | max_backward: 0.0337 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.3706 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6291.3 | uss_MB: 5817.7 | child_rss_MB: 3186.9 | allocated_MB: 207.0 | reserved_MB: 2746.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:22:33,947] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 45.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:33,963] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 13:22:34,164] INFO : Running validation on rank0 +rank0 [2026-10-28 13:22:34,165] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:22:34,165] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:22:35,269] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 13:22:35,414] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:22:35,545] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:22:35,619] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:22:35,688] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:22:35,755] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:22:35,825] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:22:35,895] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:22:36,007] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:22:36,141] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:22:36,273] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:22:36,344] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:22:36,414] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:22:36,482] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:22:36,552] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:22:36,622] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:22:36,720] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:22:36,726] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:22:36,729] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:22:36,730] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:22:36,733] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:22:36,735] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:22:36,736] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:22:36,739] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:22:36,741] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:22:36,743] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:22:36,745] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:22:36,748] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:22:36,749] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:22:36,751] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:22:36,754] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:22:36,755] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:22:42,519] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.5248 | Valid Loss=8.0991 | Train Loss Unweighted=2.6023 | Valid Loss Unweighted=2.7077 | Stale=0 | ETA=2.7m +rank0 [2026-10-28 13:22:42,529] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:42,590] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:22:42,594] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:42,774] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:44,438] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:45,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:45,734] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:46,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:46,613] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:46,616] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:46,674] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:22:46,677] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:46,817] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:48,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:48,862] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:49,497] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:50,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:50,275] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:50,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:50,332] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:22:50,335] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:22:50,468] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:52,053] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:52,579] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:53,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:53,711] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:53,953] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:53,960] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:54,212] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:54,673] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:56,120] INFO : Plotted jets +rank0 [2026-10-28 13:22:57,230] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:57,263] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:57,491] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:57,872] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:59,266] INFO : Plotted jets +rank0 [2026-10-28 13:23:00,888] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:00,921] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 13:23:01,124] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 13:23:01,544] INFO : Plotted number of elements +rank0 [2026-10-28 13:23:02,945] INFO : Plotted jets +rank0 [2026-10-28 13:23:04,052] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:15,375] INFO : Step 49100/50000 rank0 | Train Loss: 8.4073 | Train Loss Unweighted: 2.7947 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4959 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1254 | Regression_eta: 0.0123 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6280 | Classification_binary: 2.4959 | Regression_cos_phi: 0.4788 | Regression_energy: 1.6888 | Regression_eta: 0.5168 | Regression_pt: 1.8102 | Regression_sin_phi: 0.7889 | LR: 1.01e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3860s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0418 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0339 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2716 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3190.8 | allocated_MB: 207.0 | reserved_MB: 2720.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:15,589] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:26,472] INFO : Step 49200/50000 rank0 | Train Loss: 8.4810 | Train Loss Unweighted: 2.7082 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4018 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1278 | Regression_eta: 0.0125 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6379 | Classification_binary: 2.4018 | Regression_cos_phi: 0.5721 | Regression_energy: 1.7203 | Regression_eta: 0.5249 | Regression_pt: 1.8606 | Regression_sin_phi: 0.7633 | LR: 1.01e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0982 | max_forward: 0.0160 | max_loss: 0.0041 | max_backward: 0.0339 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2738 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3191.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 29.0 +rank0 [2026-10-28 13:23:26,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:37,504] INFO : Step 49300/50000 rank0 | Train Loss: 7.6860 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4217 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1142 | Regression_eta: 0.0085 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6251 | Classification_binary: 2.4217 | Regression_cos_phi: 0.3375 | Regression_energy: 1.5371 | Regression_eta: 0.3577 | Regression_pt: 1.6928 | Regression_sin_phi: 0.7141 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3894s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0163 | max_loss: 0.0034 | max_backward: 0.0339 | max_optimizer_step: 0.1432 | max_tensorboard_logging: 0.2959 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6478.5 | uss_MB: 6007.1 | child_rss_MB: 3188.8 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:23:37,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:48,602] INFO : Step 49400/50000 rank0 | Train Loss: 7.6091 | Train Loss Unweighted: 2.5949 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3211 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1145 | Regression_eta: 0.0086 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6204 | Classification_binary: 2.3211 | Regression_cos_phi: 0.4012 | Regression_energy: 1.5413 | Regression_eta: 0.3598 | Regression_pt: 1.7089 | Regression_sin_phi: 0.6564 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1013 | max_loss: 0.0034 | max_backward: 0.0330 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2981 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6478.5 | uss_MB: 6007.2 | child_rss_MB: 3189.6 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:48,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:23:59,736] INFO : Step 49500/50000 rank0 | Train Loss: 7.5559 | Train Loss Unweighted: 2.5350 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2601 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1169 | Regression_eta: 0.0099 | Regression_pt: 0.1152 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5858 | Classification_binary: 2.2601 | Regression_cos_phi: 0.4476 | Regression_energy: 1.5743 | Regression_eta: 0.4159 | Regression_pt: 1.6843 | Regression_sin_phi: 0.5880 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4292s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0345 | max_loss: 0.0036 | max_backward: 0.0337 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6478.5 | uss_MB: 6007.2 | child_rss_MB: 3189.5 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:23:59,949] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:10,807] INFO : Step 49600/50000 rank0 | Train Loss: 8.0050 | Train Loss Unweighted: 2.6557 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3509 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1338 | Regression_eta: 0.0094 | Regression_pt: 0.1294 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6090 | Classification_binary: 2.3509 | Regression_cos_phi: 0.3674 | Regression_energy: 1.8017 | Regression_eta: 0.3950 | Regression_pt: 1.8921 | Regression_sin_phi: 0.5888 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4298s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0018 | max_forward: 0.0631 | max_loss: 0.0034 | max_backward: 0.0391 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.3005 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6361.6 | uss_MB: 5892.8 | child_rss_MB: 3194.0 | allocated_MB: 210.7 | reserved_MB: 2816.0 | inactive_split_MB: 27.3 +rank0 [2026-10-28 13:24:11,021] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:22,005] INFO : Step 49700/50000 rank0 | Train Loss: 7.9377 | Train Loss Unweighted: 2.6360 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3415 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1260 | Regression_eta: 0.0085 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.6018 | Classification_binary: 2.3415 | Regression_cos_phi: 0.5167 | Regression_energy: 1.6969 | Regression_eta: 0.3571 | Regression_pt: 1.8301 | Regression_sin_phi: 0.5936 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0639 | max_forward: 0.0166 | max_loss: 0.0035 | max_backward: 0.0344 | max_optimizer_step: 0.1438 | max_tensorboard_logging: 0.2782 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 6361.6 | uss_MB: 5891.2 | child_rss_MB: 3193.2 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 25.0 +rank0 [2026-10-28 13:24:22,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 128.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:33,261] INFO : Step 49800/50000 rank0 | Train Loss: 8.4460 | Train Loss Unweighted: 2.7490 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.4432 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1272 | Regression_eta: 0.0128 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.7289 | Classification_binary: 2.4432 | Regression_cos_phi: 0.4549 | Regression_energy: 1.7122 | Regression_eta: 0.5367 | Regression_pt: 1.8568 | Regression_sin_phi: 0.7133 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3658s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0165 | max_loss: 0.0035 | max_backward: 0.0336 | max_optimizer_step: 0.1492 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 6361.6 | uss_MB: 5891.4 | child_rss_MB: 3190.7 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:33,474] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 49.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:44,482] INFO : Step 49900/50000 rank0 | Train Loss: 8.0574 | Train Loss Unweighted: 2.7208 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.4266 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1258 | Regression_eta: 0.0098 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5980 | Classification_binary: 2.4266 | Regression_cos_phi: 0.4255 | Regression_energy: 1.6932 | Regression_eta: 0.4093 | Regression_pt: 1.8224 | Regression_sin_phi: 0.6823 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4219s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0160 | max_loss: 0.0035 | max_backward: 0.1391 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6361.6 | uss_MB: 5891.5 | child_rss_MB: 3216.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:44,696] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 50.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 51.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:55,759] INFO : Step 50000/50000 rank0 | Train Loss: 7.7472 | Train Loss Unweighted: 2.6151 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.3250 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1245 | Regression_eta: 0.0077 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 33.9536 | Classification_binary: 1.0000 | Regression_cos_phi: 54.3932 | Regression_energy: 13.4624 | Regression_eta: 41.9113 | Regression_pt: 14.6249 | Regression_sin_phi: 79.1550 | Task Weighted Losses: Classification: 0.5380 | Classification_binary: 2.3250 | Regression_cos_phi: 0.4189 | Regression_energy: 1.6767 | Regression_eta: 0.3211 | Regression_pt: 1.8501 | Regression_sin_phi: 0.6174 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4850s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0162 | max_loss: 0.0034 | max_backward: 0.0638 | max_optimizer_step: 0.1436 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6361.6 | uss_MB: 5890.4 | child_rss_MB: 3190.4 | allocated_MB: 207.0 | reserved_MB: 2816.0 | inactive_split_MB: 27.0 +rank0 [2026-10-28 13:24:55,971] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 127.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 5% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 96.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 46.0°C 84.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 52.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 41.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:55,988] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 13:24:56,070] INFO : Running validation on rank0 +rank0 [2026-10-28 13:24:56,070] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:24:56,070] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:24:57,147] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 13:24:57,293] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:24:57,424] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:24:57,499] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:24:57,566] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:24:57,635] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:24:57,705] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:24:57,774] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:24:57,887] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:24:58,020] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:24:58,152] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:24:58,223] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:24:58,293] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:24:58,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:24:58,433] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:24:58,503] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:24:58,602] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:24:58,608] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:24:58,611] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:24:58,614] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:24:58,615] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:24:58,617] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:24:58,620] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:24:58,621] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:24:58,624] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:24:58,626] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:24:58,627] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:24:58,630] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:24:58,633] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:24:58,633] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:24:58,636] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:24:58,639] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:25:04,409] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.7472 | Valid Loss=8.0861 | Train Loss Unweighted=2.6151 | Valid Loss Unweighted=2.7040 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 13:25:04,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:04,481] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:25:04,485] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:04,598] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:06,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:06,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:07,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:07,769] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:08,002] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:25:08,005] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:08,061] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:25:08,079] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:08,280] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:09,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:10,250] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:10,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:11,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:11,622] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-10-28 13:25:11,625] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:25:11,679] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:25:11,681] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:25:11,817] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:25:13,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:25:13,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:25:14,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:25:14,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:25:15,244] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:25:15,252] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:15,508] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:25:15,968] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:17,411] INFO : Plotted jets +rank0 [2026-10-28 13:25:18,537] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:18,570] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:18,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:25:19,175] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:20,568] INFO : Plotted jets +rank0 [2026-10-28 13:25:22,221] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:22,254] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 13:25:22,456] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_attention_seed12345_20261028_111202_052283/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 13:25:22,873] INFO : Plotted number of elements +rank0 [2026-10-28 13:25:24,275] INFO : Plotted jets +rank0 [2026-10-28 13:25:25,376] INFO : Plotted jet ratio +rank0 [2026-10-28 13:25:25,866] INFO : Training completed. Total time on device 0: 132.873min diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth new file mode 100644 index 0000000000000000000000000000000000000000..4f68a67098142ca77c8f63e392161efd73000515 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad2caa8fcc5adf4a316b44231e6bddbf83606b701a566b8f3f57876b756ab2e +size 103962061 diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_1000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_1000.json new file mode 100644 index 0000000000000000000000000000000000000000..0cd8c139d5855b4ae75ffcfb9663a44c4ecdaefe --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_1000.json @@ -0,0 +1 @@ +{"epoch": 0.04737540269092287, "train": {"Classification_binary": 3.8990707397460938, "Classification": 0.035637266933918, "Regression_pt": 0.19362199306488037, "Regression_eta": 0.014590589329600334, "Regression_sin_phi": 0.011554048396646976, "Regression_cos_phi": 0.011099638417363167, "Regression_energy": 0.1933346688747406, "Total": 13.095527648925781, "Total_unweighted": 4.358908944763243}, "train_group": {}, "valid": {"Classification_binary": 3.8414233525594077, "Classification": 0.036764475206534065, "Regression_pt": 0.2121631701787313, "Regression_eta": 0.015537365029255549, "Regression_sin_phi": 0.011623871823151907, "Regression_cos_phi": 0.011580880731344223, "Regression_energy": 0.2154188553492228, "Total": 13.752056121826172, "Total_unweighted": 4.344511970877647, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03806482255458832, "diagnostic/loss/clic_pf/Classification_binary": 3.825500965118408, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.011023974977433681, "diagnostic/loss/clic_pf/Regression_energy": 0.221063494682312, "diagnostic/loss/clic_pf/Regression_eta": 0.01407491322606802, "diagnostic/loss/clic_pf/Regression_pt": 0.21819530427455902, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.011571750044822693, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04307243227958679, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.23697529733181, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05862605571746826, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5337716937065125, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.01155327633023262, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.24028979241847992, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08039781451225281, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5381499528884888, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9465482234954834, "metrics/particle/class_1/pid_accuracy": 0.988470733165741, "metrics/particle/class_2/efficiency": 0.18673883378505707, "metrics/particle/class_2/pid_accuracy": 0.27053138613700867, "metrics/particle/class_3/efficiency": 0.6697489023208618, "metrics/particle/class_3/pid_accuracy": 0.9847778081893921, "metrics/particle/class_4/efficiency": 0.9360465407371521, "metrics/particle/class_4/pid_accuracy": 0.05590062215924263, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.27402862906455994, "metrics/particle/count/bias_mean": -2.4790849685668945, "metrics/particle/count/mae": 4.928758144378662, "metrics/particle/count/prediction_mean": 67.520263671875, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.1019284725189209, "metrics/particle/event/energy_response_mean": 1.0083897113800049, "metrics/particle/event/met_abs_error": 11.856230735778809, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08604035526514053, "metrics/particle/event/scalar_pt_response_mean": 1.0089964866638184, "metrics/particle/event/vector_pt_closure": 0.0933067575097084, "metrics/particle/matched/delta_eta_abs_mean": 0.018284516409039497, "metrics/particle/matched/delta_phi_abs_mean": 0.012930265627801418, "metrics/particle/matched/delta_r_mean": 0.024876335635781288, "metrics/particle/matched/energy_relative_abs_mean": 0.09242895245552063, "metrics/particle/matched/pid_accuracy": 0.9572504162788391, "metrics/particle/matched/pt_relative_abs_mean": 0.09251563251018524, "metrics/particle/matching/duplicate_fraction": 0.032902251929044724, "metrics/particle/matching/efficiency": 0.7637979984283447, "metrics/particle/matching/f1": 0.7775670886039734, "metrics/particle/matching/purity": 0.7918416857719421, "metrics/particle/matching/target_coverage_dr0p05": 0.6942175030708313, "metrics/particle/matching/target_coverage_dr0p10": 0.7910718321800232, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6854779124259949, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8128460645675659, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.028551265597343445, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.09155457466840744, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0919623151421547, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.026948336511850357, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1327119767665863, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003870967775583267, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.414741188287735, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.12259891629219055, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2683571577072144, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7750.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.4123134911060333, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5829211473464966, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.525311827659607, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.383320689201355, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.16375546157360077, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.175590515136719, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.024612940847873688, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.2311294078826904, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.09161189198493958, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.023287296295166, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10076.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7209972739219666, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.8568902015686035, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 13.691442489624023, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 6.146328926086426, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.35266897082328796, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.12877094745635986, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 11.773441314697266, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5962091684341431, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 20.303741455078125, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.034882012754678726, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.10841716825962067, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.11044052243232727, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04051964730024338, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15468567609786987, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.013860014267265797, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 1.0846093893051147, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10030455142259598, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.6470980644226074, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1443.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.9165738821029663, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.9607253074645996, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.8393049240112305, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 4.661311149597168, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3251195251941681, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 44.825618743896484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.10739242285490036, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 4.455022811889648, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.38852739334106445, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 14.239984512329102, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2719.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.4697265625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 4.077052593231201, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 14.627105712890625, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 2.4037253856658936, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.026171579957008362, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.2747029960155487, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 14.340423583984375, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.9443539381027222, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 33.27421569824219, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.022788211703300476, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09172222763299942, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0918898954987526, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.024285871535539627, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1342637687921524, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0011039355304092169, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.144220232963562, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.049769606441259384, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.7058794498443604, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18117.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.2219427227973938, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.164404273033142, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.8631011247634888, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.733065664768219, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10998532176017761, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.183966636657715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.006107665132731199, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.4585585296154022, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.08824223279953003, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 6.765958309173584, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21121.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8252254724502563, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.025109976530075073, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 16.236984252929688, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 13.121026992797852, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.0208544731140137, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.9497846364974976, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 8.25689697265625, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.1803932189941406, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 43.57715606689453, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02506265603005886, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.09255677461624146, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.09289062023162842, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.02589917927980423, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13498511910438538, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002563163638114929, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.27067646384239197, "metrics/particle/momentum/all/accepted/pz_bias": 0.07310717552900314, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.2206052541732788, "metrics/particle/momentum/all/accepted/samples": 27310.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.33418703079223633, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.4327486753463745, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.378602147102356, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.241170048713684, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14320674538612366, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.17729949951172, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01972520351409912, "metrics/particle/momentum/all/assigned/pz_abs_mean": 1.0084702968597412, "metrics/particle/momentum/all/assigned/pz_bias": 0.11331676691770554, "metrics/particle/momentum/all/assigned/pz_residual_rms": 7.229177474975586, "metrics/particle/momentum/all/assigned/samples": 33916.0, "metrics/particle/momentum/all/efficiency": 0.7633822560310364, "metrics/particle/momentum/all/event/energy_bias": 2.245272636413574, "metrics/particle/momentum/all/event/p3_error_mean": 32.43437576293945, "metrics/particle/momentum/all/event/pt_vector_error_mean": 16.421791076660156, "metrics/particle/momentum/all/event/px_bias": -1.6420140266418457, "metrics/particle/momentum/all/event/py_bias": -1.5463106632232666, "metrics/particle/momentum/all/event/pz_abs_mean": 24.456525802612305, "metrics/particle/momentum/all/event/pz_bias": 4.720956325531006, "metrics/particle/momentum/all/event/pz_residual_rms": 55.36887741088867, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.016796991229057312, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.05541502684354782, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.056478943675756454, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.04711444675922394, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.08121132850646973, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010025062365457416, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.2040795385837555, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.05294829607009888, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9161773324012756, "metrics/particle/momentum/class_1/accepted/samples": 15960.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07256072759628296, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.205613374710083, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.2144078016281128, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1894703060388565, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03656729683279991, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.095425844192505, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.008475078269839287, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.5213508009910583, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.120496466755867, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 7.840444564819336, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9458898901939392, "metrics/particle/momentum/class_1/event/energy_bias": 28.235673904418945, "metrics/particle/momentum/class_1/event/p3_error_mean": 23.98670196533203, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 12.422161102294922, "metrics/particle/momentum/class_1/event/px_bias": -2.439403533935547, "metrics/particle/momentum/class_1/event/py_bias": -0.6348309516906738, "metrics/particle/momentum/class_1/event/pz_abs_mean": 17.70880889892578, "metrics/particle/momentum/class_1/event/pz_bias": 4.915200710296631, "metrics/particle/momentum/class_1/event/pz_residual_rms": 51.55934143066406, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.047902911901474, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.23294910788536072, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.22892192006111145, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.07418162375688553, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2677052617073059, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.05477031692862511, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.246770143508911, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.20201906561851501, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 5.297806262969971, "metrics/particle/momentum/class_2/accepted/samples": 566.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.4395986795425415, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.1880803108215332, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.467650055885315, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7425264716148376, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.5547671914100647, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 8.612711906433105, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.12150776386260986, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.9019904136657715, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08694250881671906, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 12.127143859863281, "metrics/particle/momentum/class_2/assigned/samples": 2255.0, "metrics/particle/momentum/class_2/efficiency": 0.19212491810321808, "metrics/particle/momentum/class_2/event/energy_bias": -25.727392196655273, "metrics/particle/momentum/class_2/event/p3_error_mean": 23.95075035095215, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.787134170532227, "metrics/particle/momentum/class_2/event/px_bias": 0.7739266157150269, "metrics/particle/momentum/class_2/event/py_bias": -0.8698076009750366, "metrics/particle/momentum/class_2/event/pz_abs_mean": 18.00094223022461, "metrics/particle/momentum/class_2/event/pz_bias": -1.7869642972946167, "metrics/particle/momentum/class_2/event/pz_residual_rms": 29.170238494873047, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0370362251996994, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1441551148891449, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14366312325000763, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": -0.0016804327024146914, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1839192807674408, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002239750698208809, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2616983950138092, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.09606150537729263, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 1.0248829126358032, "metrics/particle/momentum/class_3/accepted/samples": 10269.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.47911182045936584, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.95190167427063, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.773829460144043, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.5923893451690674, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20871946215629578, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.717458724975586, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01762145385146141, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9877985119819641, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.11440584808588028, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.287521839141846, "metrics/particle/momentum/class_3/assigned/samples": 14244.0, "metrics/particle/momentum/class_3/efficiency": 0.6662989854812622, "metrics/particle/momentum/class_3/event/energy_bias": 6.35910701751709, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.351461410522461, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.594386577606201, "metrics/particle/momentum/class_3/event/px_bias": -0.7006552219390869, "metrics/particle/momentum/class_3/event/py_bias": 0.1296381801366806, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.721896171569824, "metrics/particle/momentum/class_3/event/pz_bias": 1.4907684326171875, "metrics/particle/momentum/class_3/event/pz_residual_rms": 21.106658935546875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01989923231303692, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.06868850439786911, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06704927980899811, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.029679400846362114, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10170742124319077, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.29400670528411865, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.09324539452791214, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.014406442642212, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08775962889194489, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.057354211807251, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.8398278951644897, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.7977948188781738, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.033149171620607376, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 7.77508544921875, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.35532236099243164, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.10345904529094696, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1714452505111694, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -4.336797714233398, "metrics/particle/momentum/class_4/event/p3_error_mean": 4.820202350616455, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 3.6781740188598633, "metrics/particle/momentum/class_4/event/px_bias": 0.42509275674819946, "metrics/particle/momentum/class_4/event/py_bias": 0.2938292920589447, "metrics/particle/momentum/class_4/event/pz_abs_mean": 2.3595762252807617, "metrics/particle/momentum/class_4/event/pz_bias": 0.2635892331600189, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.917102813720703, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01252601109445095, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.04467141255736351, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.044801488518714905, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.025827448815107346, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07258965075016022, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.43297845125198364, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.10840074717998505, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.2397279739379883, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04086396470665932, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.08255346864461899, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.08225980401039124, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.051641978323459625, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.37044766545295715, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.8445644378662109, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29115408658981323, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.1840434074401855, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -2.285318374633789, "metrics/particle/momentum/class_5/event/p3_error_mean": 3.316427230834961, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 2.475370168685913, "metrics/particle/momentum/class_5/event/px_bias": 0.2990253269672394, "metrics/particle/momentum/class_5/event/py_bias": -0.46513956785202026, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.7636744976043701, "metrics/particle/momentum/class_5/event/pz_bias": -0.16163769364356995, "metrics/particle/momentum/class_5/event/pz_residual_rms": 6.288617134094238, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3547377586364746, "step_valid_time": 440.93380641937256, "step_total_time": 442.2885446548462, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.010843457520156, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1494904763371545, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3007, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2764, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2682, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9167495854063018, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9191885600266045, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9179674526735304, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0808114399733954, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8895522388059701, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8919188560026604, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8907339754234473, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9167495854063018, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999443668170243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1617199610156228, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2220, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1988, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1950, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8823790501553485, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8954954954954955, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8888888888888888, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1045045045045045, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8655126498002663, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8783783783783784, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8718980549966465, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8823790501553485, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046897265333308, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18269616454756465, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1272, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1078, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8389105058365759, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8474842767295597, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8431755964020337, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.15251572327044025, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8124513618677043, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8207547169811321, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8165819319515056, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8389105058365759} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_10000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..d33ea2d88f537af704c7f6e853d8b0a0568e25bf --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.47375402690922874, "train": {"Classification_binary": 2.9295153617858887, "Classification": 0.02634306624531746, "Regression_pt": 0.17762845754623413, "Regression_eta": 0.01764829456806183, "Regression_sin_phi": 0.012765102088451385, "Regression_cos_phi": 0.010787692852318287, "Regression_energy": 0.1760053187608719, "Total": 11.454862594604492, "Total_unweighted": 3.3506932938471437}, "train_group": {}, "valid": {"Classification_binary": 2.792769432067871, "Classification": 0.02300037443637848, "Regression_pt": 0.1458946963151296, "Regression_eta": 0.011373197038968405, "Regression_sin_phi": 0.008751060192783674, "Regression_cos_phi": 0.00891956128180027, "Regression_energy": 0.14976045489311218, "Total": 9.67567507425944, "Total_unweighted": 3.1404687762260437, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.023465821519494057, "diagnostic/loss/clic_pf/Classification_binary": 2.8012442588806152, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008470636792480946, "diagnostic/loss/clic_pf/Regression_energy": 0.15428566932678223, "diagnostic/loss/clic_pf/Regression_eta": 0.010757345706224442, "diagnostic/loss/clic_pf/Regression_pt": 0.15101498365402222, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008583923801779747, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02889256738126278, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2001912146806717, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0444461964070797, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4516746401786804, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.002446418395265937, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19399988651275635, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06639812141656876, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4541154205799103, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9631264209747314, "metrics/particle/class_1/pid_accuracy": 0.9866052865982056, "metrics/particle/class_2/efficiency": 0.39557960629463196, "metrics/particle/class_2/pid_accuracy": 0.6818699836730957, "metrics/particle/class_3/efficiency": 0.726004958152771, "metrics/particle/class_3/pid_accuracy": 0.9835821986198425, "metrics/particle/class_4/efficiency": 0.9195736646652222, "metrics/particle/class_4/pid_accuracy": 0.34878820180892944, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.6146044731140137, "metrics/particle/count/bias_mean": -1.445751667022705, "metrics/particle/count/mae": 3.694117546081543, "metrics/particle/count/prediction_mean": 68.55359649658203, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05492498725652695, "metrics/particle/event/energy_response_mean": 1.0033295154571533, "metrics/particle/event/met_abs_error": 5.932789325714111, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0442778542637825, "metrics/particle/event/scalar_pt_response_mean": 1.0069209337234497, "metrics/particle/event/vector_pt_closure": 0.04832806438207626, "metrics/particle/matched/delta_eta_abs_mean": 0.012672977522015572, "metrics/particle/matched/delta_phi_abs_mean": 0.013309502974152565, "metrics/particle/matched/delta_r_mean": 0.02048318460583687, "metrics/particle/matched/energy_relative_abs_mean": 0.08435598015785217, "metrics/particle/matched/pid_accuracy": 0.9641040563583374, "metrics/particle/matched/pt_relative_abs_mean": 0.07450270652770996, "metrics/particle/matching/duplicate_fraction": 0.03173891827464104, "metrics/particle/matching/efficiency": 0.8131261467933655, "metrics/particle/matching/f1": 0.8216108679771423, "metrics/particle/matching/purity": 0.8302744626998901, "metrics/particle/matching/target_coverage_dr0p05": 0.752257227897644, "metrics/particle/matching/target_coverage_dr0p10": 0.8435466289520264, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.725562334060669, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8543683886528015, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023112930357456207, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07918570190668106, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07267535477876663, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.005161960609257221, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12857289612293243, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0039492580108344555, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3323395252227783, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.014633494429290295, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.246854543685913, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8356.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3152126669883728, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.007401943206787, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2391586303710938, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.114418625831604, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.130121648311615, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 25.714736938476562, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017128953710198402, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8162945508956909, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.009725705720484257, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1043131351470947, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10275.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7773746252059937, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.6851414442062378, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.833520412445068, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.6234865188598633, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.36832594871520996, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3916422724723816, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.670105934143066, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.391396164894104, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.987598419189453, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030678745359182358, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0863572433590889, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0839712843298912, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.007165106944739819, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13984517753124237, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019607843831181526, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8391458988189697, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07314301282167435, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.868798017501831, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1734.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.744066059589386, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4603058099746704, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 122040.1796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 122040.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25244832038879395, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 6407811.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08342400938272476, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7967236042022705, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.15418009459972382, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.284501075744629, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2757.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.564453125, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7233837842941284, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.21559476852417, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1833815574645996, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.027080275118350983, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.011305175721645355, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.058727264404297, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.1177916526794434, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.70938491821289, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018686627969145775, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08692307770252228, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07523292303085327, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013253049924969673, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13225671648979187, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008951137424446642, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12084902077913284, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03855389356613159, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5962527990341187, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18992.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15046904981136322, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.730373203754425, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5368797183036804, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.43565574288368225, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08653891086578369, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.210892200469971, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034969926346093416, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2943729758262634, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.026016440242528915, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 4.044379234313965, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21447.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8650815486907959, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.260075569152832, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.354434967041016, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.510082244873047, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.3864262104034424, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.6516169309616089, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.976736545562744, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.08664306998252869, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 25.856130599975586, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.020673438906669617, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0846661925315857, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07501908391714096, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01056528277695179, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13167716562747955, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002888384507969022, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2244437336921692, "metrics/particle/momentum/all/accepted/pz_bias": -0.025021081790328026, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.081466555595398, "metrics/particle/momentum/all/accepted/samples": 29082.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24702893197536469, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8712965250015259, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 9759.244140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 9759.1298828125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11279329657554626, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 1811969.0, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013950520195066929, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6500017642974854, "metrics/particle/momentum/all/assigned/pz_bias": -0.0009561891201883554, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.305159091949463, "metrics/particle/momentum/all/assigned/samples": 34479.0, "metrics/particle/momentum/all/efficiency": 0.8129140734672546, "metrics/particle/momentum/all/event/energy_bias": -0.14844977855682373, "metrics/particle/momentum/all/event/p3_error_mean": 16.91196060180664, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.789876937866211, "metrics/particle/momentum/all/event/px_bias": -2.0451807975769043, "metrics/particle/momentum/all/event/py_bias": -0.27127984166145325, "metrics/particle/momentum/all/event/pz_abs_mean": 12.645304679870605, "metrics/particle/momentum/all/event/pz_bias": -0.18696138262748718, "metrics/particle/momentum/all/event/pz_residual_rms": 29.83121109008789, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010570034384727478, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04025952145457268, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02279624156653881, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006427675485610962, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06395862251520157, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014162561856210232, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12136714905500412, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.01860167644917965, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8270249366760254, "metrics/particle/momentum/class_1/accepted/samples": 16240.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05107109248638153, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1464136838912964, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12333180755376816, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09428040683269501, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02382504567503929, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.8658230304718018, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00497836759313941, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29588258266448975, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.020593641325831413, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.808159828186035, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9624844193458557, "metrics/particle/momentum/class_1/event/energy_bias": 6.313235282897949, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.844032287597656, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.659322738647461, "metrics/particle/momentum/class_1/event/px_bias": -1.9148242473602295, "metrics/particle/momentum/class_1/event/py_bias": -0.4854697585105896, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.593207359313965, "metrics/particle/momentum/class_1/event/pz_bias": 0.9027881026268005, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.53213882446289, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043599069118499756, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20181512832641602, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20412595570087433, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04267367348074913, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24440471827983856, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0385604128241539, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7417824268341064, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.11063733696937561, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7306435108184814, "metrics/particle/momentum/class_2/accepted/samples": 1167.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.929577648639679, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.770554780960083, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 132099.8125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 132099.296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3694542646408081, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 6666741.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08912446349859238, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2569940090179443, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05137386918067932, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.303318500518799, "metrics/particle/momentum/class_2/assigned/samples": 2547.0, "metrics/particle/momentum/class_2/efficiency": 0.39613035321235657, "metrics/particle/momentum/class_2/event/energy_bias": -9.586657524108887, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.059375762939453, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.720560073852539, "metrics/particle/momentum/class_2/event/px_bias": -0.2758151590824127, "metrics/particle/momentum/class_2/event/py_bias": 0.15013232827186584, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.218009948730469, "metrics/particle/momentum/class_2/event/pz_bias": -0.38961175084114075, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.86644172668457, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03334936872124672, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13872011005878448, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1387641727924347, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.021866708993911743, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18046163022518158, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001432793098501861, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21452271938323975, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0240351390093565, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.758028507232666, "metrics/particle/momentum/class_3/accepted/samples": 11167.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3618612587451935, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.75276780128479, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9661378860473633, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8197240829467773, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1734757125377655, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 27.876985549926758, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.011505339294672012, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6128969788551331, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.028856396675109863, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.749685049057007, "metrics/particle/momentum/class_3/assigned/samples": 14515.0, "metrics/particle/momentum/class_3/efficiency": 0.7245652675628662, "metrics/particle/momentum/class_3/event/energy_bias": 4.483280181884766, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.417037963867188, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.677783012390137, "metrics/particle/momentum/class_3/event/px_bias": -0.11112557351589203, "metrics/particle/momentum/class_3/event/py_bias": 0.1880902498960495, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.475116729736328, "metrics/particle/momentum/class_3/event/pz_bias": -0.46803680062294006, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.368795394897461, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014034595340490341, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0607731007039547, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06072140857577324, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.022390101104974747, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10503286868333817, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.28990206122398376, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.035313986241817474, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7775717973709106, "metrics/particle/momentum/class_4/accepted/samples": 328.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09021373093128204, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.44586455821990967, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3950468599796295, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3457479774951935, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.07182320207357407, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.539367914199829, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3615834414958954, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.03477323055267334, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0383448600769043, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9060773253440857, "metrics/particle/momentum/class_4/event/energy_bias": -0.6563747525215149, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.7670319080352783, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1848710775375366, "metrics/particle/momentum/class_4/event/px_bias": 0.10642587393522263, "metrics/particle/momentum/class_4/event/py_bias": -0.10251383483409882, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0748875141143799, "metrics/particle/momentum/class_4/event/pz_bias": 0.02881672978401184, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.5665242671966553, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009287789463996887, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02171226404607296, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02101733535528183, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00635985191911459, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0674748495221138, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18303696811199188, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.09152622520923615, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5209193229675293, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.01581883244216442, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.030567651614546776, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.029758447781205177, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.004697709809988737, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10981282591819763, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5292727947235107, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4387676417827606, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.04830265045166, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.7019329071044922, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.221952199935913, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.8153141736984253, "metrics/particle/momentum/class_5/event/px_bias": 0.15015846490859985, "metrics/particle/momentum/class_5/event/py_bias": -0.02151881903409958, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.764706552028656, "metrics/particle/momentum/class_5/event/pz_bias": -0.26091769337654114, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.773090362548828, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6362125873565674, "step_valid_time": 4332.067195415497, "step_total_time": 4333.703408718109, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012422863083572, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07800278709575248, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2879, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2866, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9548922056384743, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9529956967891426, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9539430086149768, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04700430321085733, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9505804311774461, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9486924859318107, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.949635520212061, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9548922056384743, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9986463639291514, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08641259253145828, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2107, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9369729249889037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9457885304659498, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9413600891861762, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05421146953405018, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9351975144252108, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9439964157706093, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9395763656633221, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9369729249889037, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9978509881114418, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08339729509740623, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1158, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1153, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9011673151750973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9011673151750973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9011673151750973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09883268482490272, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8972762645914397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8972762645914397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8972762645914397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9011673151750973} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_11000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_11000.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6b7892d7c1f16f3814c70d52c5e9eeabf37023 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_11000.json @@ -0,0 +1 @@ +{"epoch": 0.5211294296001516, "train": {"Classification_binary": 2.743016481399536, "Classification": 0.02287846803665161, "Regression_pt": 0.1407506763935089, "Regression_eta": 0.010725436732172966, "Regression_sin_phi": 0.008865421637892723, "Regression_cos_phi": 0.009624478407204151, "Regression_energy": 0.14286889135837555, "Total": 9.469865798950195, "Total_unweighted": 3.078729853965342}, "train_group": {}, "valid": {"Classification_binary": 2.769200007120768, "Classification": 0.022487469017505646, "Regression_pt": 0.14364968736966452, "Regression_eta": 0.011680938303470612, "Regression_sin_phi": 0.008785955607891083, "Regression_cos_phi": 0.008881565804282824, "Regression_energy": 0.14765686790148416, "Total": 9.582330067952475, "Total_unweighted": 3.112342491125067, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02306261658668518, "diagnostic/loss/clic_pf/Classification_binary": 2.775512456893921, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008434096351265907, "diagnostic/loss/clic_pf/Regression_energy": 0.15208250284194946, "diagnostic/loss/clic_pf/Regression_eta": 0.011188717558979988, "diagnostic/loss/clic_pf/Regression_pt": 0.1487245261669159, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008471764624118805, "diagnostic/regression/clic_pf/energy_pred_mean": 0.030187634751200676, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2008775919675827, "diagnostic/regression/clic_pf/energy_residual_mean": 0.045741256326436996, "diagnostic/regression/clic_pf/energy_residual_rms": 0.449736624956131, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.006200703326612711, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1938284933567047, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07504522800445557, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4529678225517273, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9623908996582031, "metrics/particle/class_1/pid_accuracy": 0.9840958118438721, "metrics/particle/class_2/efficiency": 0.3988497853279114, "metrics/particle/class_2/pid_accuracy": 0.6940910220146179, "metrics/particle/class_3/efficiency": 0.7356611490249634, "metrics/particle/class_3/pid_accuracy": 0.9825085401535034, "metrics/particle/class_4/efficiency": 0.9234496355056763, "metrics/particle/class_4/pid_accuracy": 0.414480596780777, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6639510989189148, "metrics/particle/count/bias_mean": 0.350326806306839, "metrics/particle/count/mae": 3.4183006286621094, "metrics/particle/count/prediction_mean": 70.34967041015625, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05588899180293083, "metrics/particle/event/energy_response_mean": 1.0032353401184082, "metrics/particle/event/met_abs_error": 6.678739070892334, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04682137817144394, "metrics/particle/event/scalar_pt_response_mean": 1.0210610628128052, "metrics/particle/event/vector_pt_closure": 0.049674779176712036, "metrics/particle/matched/delta_eta_abs_mean": 0.01866886019706726, "metrics/particle/matched/delta_phi_abs_mean": 0.014107638038694859, "metrics/particle/matched/delta_r_mean": 0.0258756335824728, "metrics/particle/matched/energy_relative_abs_mean": 0.08678934723138809, "metrics/particle/matched/pid_accuracy": 0.9637597799301147, "metrics/particle/matched/pt_relative_abs_mean": 0.0754796639084816, "metrics/particle/matching/duplicate_fraction": 0.0312909372150898, "metrics/particle/matching/efficiency": 0.8172531723976135, "metrics/particle/matching/f1": 0.8152132630348206, "metrics/particle/matching/purity": 0.8131834268569946, "metrics/particle/matching/target_coverage_dr0p05": 0.7414074540138245, "metrics/particle/matching/target_coverage_dr0p10": 0.8506708741188049, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7279899716377258, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.86048424243927, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.029023775830864906, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08306309580802917, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07555432617664337, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017476510256528854, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1321292221546173, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0038058990612626076, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3448982834815979, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01737060397863388, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2488538026809692, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8408.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3208983540534973, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1617120504379272, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.206181287765503, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0843018293380737, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13565151393413544, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 18.011791229248047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01432938501238823, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8021344542503357, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04216250404715538, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2417216300964355, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10468.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7822123169898987, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.396900475025177, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.150764465332031, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.3417534828186035, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2473079264163971, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.09661699086427689, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.007287979125977, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.2075613737106323, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.661857604980469, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03597358986735344, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0891658291220665, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08433423191308975, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0114785460755229, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14109869301319122, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.012702078558504581, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8172335624694824, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.016895949840545654, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.793417453765869, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1732.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8453238606452942, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6409895420074463, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.633655309677124, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.4458115100860596, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2751377522945404, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 49.64287567138672, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08402203768491745, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.1848621368408203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0927339568734169, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.091997146606445, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2904.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5638020634651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -4.272645473480225, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.985608100891113, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1584720611572266, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03525654971599579, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03258673846721649, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.830449104309082, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.2626917362213135, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.46003246307373, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023511186242103577, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08835897594690323, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0752916932106018, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01737927459180355, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13278397917747498, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006308153388090432, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12687373161315918, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.036415647715330124, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5575439929962158, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19023.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.16445237398147583, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.755820631980896, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5746226906776428, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.47190800309181213, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09316942095756531, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.178342819213867, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0030964044854044914, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.29967981576919556, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.018871668726205826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 4.0270676612854, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21638.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8664935827255249, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 4.294441223144531, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.349123001098633, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.43672513961792, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.390725702047348, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.0490361750125885, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.095773220062256, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1775662899017334, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 25.430715560913086, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025840673595666885, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08688004314899445, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07590445131063461, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017056861892342567, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13310456275939941, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0022631415631622076, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23073318600654602, "metrics/particle/momentum/all/accepted/pz_bias": -0.02775857038795948, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0563443899154663, "metrics/particle/momentum/all/accepted/samples": 29163.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2677065134048462, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9506050944328308, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0171986818313599, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9016925096511841, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12096543610095978, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 18.801239013671875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013167666271328926, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6892331838607788, "metrics/particle/momentum/all/assigned/pz_bias": 0.008634999394416809, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.649014472961426, "metrics/particle/momentum/all/assigned/samples": 35010.0, "metrics/particle/momentum/all/efficiency": 0.8151782155036926, "metrics/particle/momentum/all/event/energy_bias": -0.37510454654693604, "metrics/particle/momentum/all/event/p3_error_mean": 16.251996994018555, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.556421279907227, "metrics/particle/momentum/all/event/px_bias": -0.17867428064346313, "metrics/particle/momentum/all/event/py_bias": 0.014994055032730103, "metrics/particle/momentum/all/event/pz_abs_mean": 12.121133804321289, "metrics/particle/momentum/all/event/pz_bias": -0.1224360466003418, "metrics/particle/momentum/all/event/pz_residual_rms": 29.49736213684082, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017754454165697098, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04489010572433472, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022854208946228027, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002852995414286852, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06453569233417511, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001419665408320725, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15256652235984802, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.043756913393735886, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.890821635723114, "metrics/particle/momentum/class_1/accepted/samples": 16201.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06012459471821785, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16018426418304443, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.16526751220226288, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.13241396844387054, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.026077164337038994, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.726888179779053, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005215432960540056, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3224390745162964, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02781328186392784, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.640851974487305, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9601730704307556, "metrics/particle/momentum/class_1/event/energy_bias": 4.226626873016357, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.995121955871582, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.487563133239746, "metrics/particle/momentum/class_1/event/px_bias": -1.8165124654769897, "metrics/particle/momentum/class_1/event/py_bias": -0.9807066917419434, "metrics/particle/momentum/class_1/event/pz_abs_mean": 9.007884979248047, "metrics/particle/momentum/class_1/event/pz_bias": -0.9868736267089844, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.95630645751953, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.042868077754974365, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19940847158432007, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2017228603363037, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02694820985198021, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2413078248500824, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028546713292598724, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5791600942611694, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.04539259895682335, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4358346462249756, "metrics/particle/momentum/class_2/accepted/samples": 1156.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.016135573387146, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.832645833492279, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9305115938186646, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3844948709011078, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3982202410697937, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.949279546737671, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08305524289608002, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4360625743865967, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.27742889523506165, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.9130859375, "metrics/particle/momentum/class_2/assigned/samples": 2697.0, "metrics/particle/momentum/class_2/efficiency": 0.392396479845047, "metrics/particle/momentum/class_2/event/energy_bias": -10.054433822631836, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.720107078552246, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.558046340942383, "metrics/particle/momentum/class_2/event/px_bias": 0.5832469463348389, "metrics/particle/momentum/class_2/event/py_bias": 0.23271138966083527, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.910284042358398, "metrics/particle/momentum/class_2/event/pz_bias": 0.24962922930717468, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.590913772583008, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.036029454320669174, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13742558658123016, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14049650728702545, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04239378497004509, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18274158239364624, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008856611675582826, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2009653002023697, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0030846833251416683, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7059683799743652, "metrics/particle/momentum/class_3/accepted/samples": 11291.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.37424397468566895, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8875192403793335, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0249106884002686, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8913158178329468, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18112245202064514, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.328767776489258, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009868420660495758, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6163958311080933, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.010119641199707985, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.440091609954834, "metrics/particle/momentum/class_3/assigned/samples": 14896.0, "metrics/particle/momentum/class_3/efficiency": 0.7326109409332275, "metrics/particle/momentum/class_3/event/energy_bias": 6.803988933563232, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.362602233886719, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.576712608337402, "metrics/particle/momentum/class_3/event/px_bias": 0.7165645360946655, "metrics/particle/momentum/class_3/event/py_bias": 0.7612267136573792, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.57734489440918, "metrics/particle/momentum/class_3/event/pz_bias": 0.5163468718528748, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.976283073425293, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.020072875544428825, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0577886663377285, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.058080997318029404, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.010489888489246368, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10617668181657791, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.28167033195495605, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.08969920873641968, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8344635367393494, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10983636975288391, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5770909190177917, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3994002044200897, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.34119468927383423, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.450253963470459, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3465825021266937, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.07762079685926437, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0367742776870728, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.7534905076026917, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9499120712280273, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.22896409034729, "metrics/particle/momentum/class_4/event/px_bias": 0.1768837571144104, "metrics/particle/momentum/class_4/event/py_bias": -0.04361313581466675, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2912997007369995, "metrics/particle/momentum/class_4/event/pz_bias": 0.3936491310596466, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.823774814605713, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.015882648527622223, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02688492275774479, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023953814059495926, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0009339256794191897, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07632046192884445, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3792888820171356, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.29516083002090454, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9960010647773743, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.015967218205332756, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.037669163197278976, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.034741662442684174, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0027288191486150026, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11662395298480988, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6328799724578857, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.5454082489013672, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.593829393386841, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.5977959632873535, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.176032304763794, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.7148379683494568, "metrics/particle/momentum/class_5/event/px_bias": 0.1611429899930954, "metrics/particle/momentum/class_5/event/py_bias": 0.04537583142518997, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8043433427810669, "metrics/particle/momentum/class_5/event/pz_bias": -0.2951876223087311, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7952630519866943, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4552404880523682, "step_valid_time": 4764.084578037262, "step_total_time": 4765.5398190021515, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036105083579365, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07196521983081983, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2871, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2863, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9522388059701492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9465875370919882, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9494047619047619, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05341246290801187, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9495854063018242, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9439498846027036, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9467592592592593, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9522388059701492, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0088931943930677, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08836602278675332, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2095, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2089, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9298712827341322, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9327693677649154, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9313180706823738, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06723063223508459, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.927208166888593, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9300979519145147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.928650811291398, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9298712827341322, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0130232836049506, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07946616925923, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1145, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8964980544747082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8992974238875878, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8978955572876072, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10070257611241218, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8910505836575876, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8938329430132709, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8924395946999221, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8964980544747082} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_12000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_12000.json new file mode 100644 index 0000000000000000000000000000000000000000..30e02ce9c36d41c919049b8bf40c51147ced78f1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_12000.json @@ -0,0 +1 @@ +{"epoch": 0.5685048322910745, "train": {"Classification_binary": 2.650594472885132, "Classification": 0.021725067868828773, "Regression_pt": 0.1348210722208023, "Regression_eta": 0.009052018634974957, "Regression_sin_phi": 0.007153885439038277, "Regression_cos_phi": 0.008021400310099125, "Regression_energy": 0.13600274920463562, "Total": 8.858095169067383, "Total_unweighted": 2.967370666563511}, "train_group": {}, "valid": {"Classification_binary": 2.7053918838500977, "Classification": 0.021821528673171997, "Regression_pt": 0.14117252826690674, "Regression_eta": 0.011074164261420568, "Regression_sin_phi": 0.008496674398581186, "Regression_cos_phi": 0.008662352338433266, "Regression_energy": 0.14480779568354288, "Total": 9.353006362915039, "Total_unweighted": 3.041426927472154, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022349564358592033, "diagnostic/loss/clic_pf/Classification_binary": 2.7165892124176025, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008192998357117176, "diagnostic/loss/clic_pf/Regression_energy": 0.14928510785102844, "diagnostic/loss/clic_pf/Regression_eta": 0.010431796312332153, "diagnostic/loss/clic_pf/Regression_pt": 0.14621539413928986, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008264795877039433, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02872394397854805, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19799236953258514, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04427756369113922, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4494277238845825, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005086122080683708, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19091373682022095, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06375841796398163, "diagnostic/regression/clic_pf/pt_residual_rms": 0.450177937746048, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9643588662147522, "metrics/particle/class_1/pid_accuracy": 0.9846230149269104, "metrics/particle/class_2/efficiency": 0.411253958940506, "metrics/particle/class_2/pid_accuracy": 0.7343021631240845, "metrics/particle/class_3/efficiency": 0.7474512457847595, "metrics/particle/class_3/pid_accuracy": 0.983245849609375, "metrics/particle/class_4/efficiency": 0.9282945990562439, "metrics/particle/class_4/pid_accuracy": 0.46033403277397156, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7326530814170837, "metrics/particle/count/bias_mean": -0.40522876381874084, "metrics/particle/count/mae": 3.355555534362793, "metrics/particle/count/prediction_mean": 69.5941162109375, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.057446617633104324, "metrics/particle/event/energy_response_mean": 1.0089067220687866, "metrics/particle/event/met_abs_error": 6.294971466064453, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04487791284918785, "metrics/particle/event/scalar_pt_response_mean": 1.0113658905029297, "metrics/particle/event/vector_pt_closure": 0.048816341906785965, "metrics/particle/matched/delta_eta_abs_mean": 0.013074514456093311, "metrics/particle/matched/delta_phi_abs_mean": 0.011909060180187225, "metrics/particle/matched/delta_r_mean": 0.019779972732067108, "metrics/particle/matched/energy_relative_abs_mean": 0.08427120000123978, "metrics/particle/matched/pid_accuracy": 0.9666541814804077, "metrics/particle/matched/pt_relative_abs_mean": 0.07401841878890991, "metrics/particle/matching/duplicate_fraction": 0.028982240706682205, "metrics/particle/matching/efficiency": 0.8243494629859924, "metrics/particle/matching/f1": 0.8267424702644348, "metrics/particle/matching/purity": 0.8291494250297546, "metrics/particle/matching/target_coverage_dr0p05": 0.7794097065925598, "metrics/particle/matching/target_coverage_dr0p10": 0.8545084595680237, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7370657324790955, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8635841608047485, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02282356470823288, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08092387020587921, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07326240837574005, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.004441659431904554, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1281418651342392, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0032821474596858025, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32391735911369324, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.010279122740030289, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1968896389007568, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8531.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2950681149959564, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.033474326133728, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3436225652694702, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.219525933265686, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12862353026866913, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 29.598512649536133, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015454022213816643, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7878379225730896, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.014438200742006302, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.255079746246338, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10418.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7936552166938782, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7809441685676575, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.148715972900391, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.153902053833008, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.264074444770813, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.20771831274032593, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.181550025939941, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3740949332714081, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.107978820800781, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.032942995429039, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08948739618062973, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0840562954545021, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.005129100754857063, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13920754194259644, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.014959723688662052, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8199503421783447, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.038855817168951035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.5466830730438232, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1738.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7764747142791748, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.91152822971344, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 4.115484237670898, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.9438633918762207, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2680228352546692, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 57.86796569824219, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08137045055627823, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.0610790252685547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.06608960777521133, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.263736724853516, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2802.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5657551884651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.5686771869659424, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.168536186218262, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2055944204330444, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1762678176164627, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.17814821004867554, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.990428924560547, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.0774834156036377, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.117061614990234, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017526010051369667, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08584553003311157, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07433303445577621, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.00823217909783125, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13063400983810425, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005737533792853355, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11100894212722778, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.018632950261235237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5606369376182556, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19172.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15078110992908478, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7312840819358826, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5201079249382019, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.412896990776062, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08513300865888596, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.355435848236084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0032903882674872875, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2888154983520508, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002095684874802828, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.974865436553955, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21578.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8732805252075195, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.2399089336395264, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.569453239440918, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.905562400817871, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.8638877272605896, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.0305759906768799, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.507106781005859, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.2552391290664673, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 24.239707946777344, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019971175119280815, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08463439345359802, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07459679245948792, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.006950629409402609, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.130441352725029, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0022078054025769234, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21455372869968414, "metrics/particle/momentum/all/accepted/pz_bias": -0.017406117171049118, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0013242959976196, "metrics/particle/momentum/all/accepted/samples": 29441.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24436047673225403, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9167908430099487, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.0561623573303223, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.9387096166610718, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11288005113601685, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 23.779720306396484, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013219150714576244, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6614432334899902, "metrics/particle/momentum/all/assigned/pz_bias": -0.008344720117747784, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.4586405754089355, "metrics/particle/momentum/all/assigned/samples": 34798.0, "metrics/particle/momentum/all/efficiency": 0.8229489922523499, "metrics/particle/momentum/all/event/energy_bias": -0.10971269011497498, "metrics/particle/momentum/all/event/p3_error_mean": 16.58428192138672, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.371850967407227, "metrics/particle/momentum/all/event/px_bias": -1.304229974746704, "metrics/particle/momentum/all/event/py_bias": -0.6447094678878784, "metrics/particle/momentum/all/event/pz_abs_mean": 12.511384963989258, "metrics/particle/momentum/all/event/pz_bias": -2.1963391304016113, "metrics/particle/momentum/all/event/pz_residual_rms": 29.51140785217285, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011370989494025707, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04137420654296875, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.023367436602711678, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006931505165994167, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06478103250265121, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012916717678308487, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11615204066038132, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00400847801938653, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7422254681587219, "metrics/particle/momentum/class_1/accepted/samples": 16258.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.052664294838905334, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18389657139778137, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.16017268598079681, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.13163764774799347, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.023236515000462532, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.053391933441162, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004919976461678743, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2903003990650177, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01622881181538105, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.549404621124268, "metrics/particle/momentum/class_1/assigned/samples": 16870.0, "metrics/particle/momentum/class_1/efficiency": 0.9635512232780457, "metrics/particle/momentum/class_1/event/energy_bias": 3.392735481262207, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.283321380615234, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.0910234451293945, "metrics/particle/momentum/class_1/event/px_bias": -1.2821242809295654, "metrics/particle/momentum/class_1/event/py_bias": -0.26987147331237793, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.440397262573242, "metrics/particle/momentum/class_1/event/pz_bias": 0.01828744262456894, "metrics/particle/momentum/class_1/event/pz_residual_rms": 27.137149810791016, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04013795405626297, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20126613974571228, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20197463035583496, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04723959043622017, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24230149388313293, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02874048985540867, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6462966203689575, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03533541038632393, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4646267890930176, "metrics/particle/momentum/class_2/accepted/samples": 1183.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9374322891235352, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7502389550209045, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8836769461631775, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.36176279187202454, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3765197694301605, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.799363374710083, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08548632264137268, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.319711446762085, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05366219952702522, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.776626110076904, "metrics/particle/momentum/class_2/assigned/samples": 2632.0, "metrics/particle/momentum/class_2/efficiency": 0.401561439037323, "metrics/particle/momentum/class_2/event/energy_bias": -6.37584924697876, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.800558090209961, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.294418811798096, "metrics/particle/momentum/class_2/event/px_bias": -0.6501766443252563, "metrics/particle/momentum/class_2/event/py_bias": -0.4831821918487549, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.137931823730469, "metrics/particle/momentum/class_2/event/pz_bias": -0.6557901501655579, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.281112670898438, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030405396595597267, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1357620358467102, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13540108501911163, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.012583681382238865, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17684558033943176, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008705492946319282, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20618748664855957, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04084773361682892, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7275922894477844, "metrics/particle/momentum/class_3/accepted/samples": 11487.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3457155227661133, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7520661354064941, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.0926499366760254, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9429640769958496, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1707565039396286, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 35.75008010864258, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010168112814426422, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6206960082054138, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.041058361530303955, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.49336576461792, "metrics/particle/momentum/class_3/assigned/samples": 14752.0, "metrics/particle/momentum/class_3/efficiency": 0.7453283071517944, "metrics/particle/momentum/class_3/event/energy_bias": 3.344679355621338, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.727563858032227, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.039180278778076, "metrics/particle/momentum/class_3/event/px_bias": 0.3957535922527313, "metrics/particle/momentum/class_3/event/py_bias": 0.07998079061508179, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.230035781860352, "metrics/particle/momentum/class_3/event/pz_bias": -1.694434404373169, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.948234558105469, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013849460519850254, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.054472073912620544, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05617256835103035, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.003334703855216503, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10278862714767456, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24237346649169922, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.023628491908311844, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8408297896385193, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1262601912021637, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.691544771194458, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.345139741897583, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.286973237991333, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 31.295381546020508, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.38622617721557617, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09253235161304474, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.7859443426132202, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.3688167929649353, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8780394792556763, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1728515625, "metrics/particle/momentum/class_4/event/px_bias": 0.1690753549337387, "metrics/particle/momentum/class_4/event/py_bias": -0.05149137228727341, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2516653537750244, "metrics/particle/momentum/class_4/event/pz_bias": 0.25337085127830505, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.7301106452941895, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00966684240847826, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.018464695662260056, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01834791712462902, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.012035052292048931, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.052077047526836395, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17511869966983795, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06666269153356552, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5037240386009216, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.009814289398491383, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.025833439081907272, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02567524090409279, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010109844617545605, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09018369019031525, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.47104889154434204, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.36378470063209534, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5171546936035156, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.10246113687753677, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0875434875488281, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6797788739204407, "metrics/particle/momentum/class_5/event/px_bias": 0.0632418766617775, "metrics/particle/momentum/class_5/event/py_bias": 0.07985479384660721, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7264715433120728, "metrics/particle/momentum/class_5/event/pz_bias": -0.11777310073375702, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7468371391296387, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4012963771820068, "step_valid_time": 5198.44295501709, "step_total_time": 5199.84425163269, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000514781642723, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06844415552567384, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2877, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2870, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.954228855721393, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9517036056897122, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9529645578005962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048296394310287795, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9519071310116086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9493880251405888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9506459092414706, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.954228855721393, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999252994836475, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08155722272313837, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2239, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2116, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9391921881935198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.945064761054042, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9421193232413179, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05493523894595802, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9369729249889037, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9428316212594908, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.939893143365984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9391921881935198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026440391006226, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0766274469560666, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1154, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1150, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8980544747081712, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9001560062402496, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8991040124659135, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0998439937597504, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8949416342412452, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8970358814352574, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8959875340864822, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8980544747081712} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_13000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_13000.json new file mode 100644 index 0000000000000000000000000000000000000000..39e84fd9666399de7122dcd40936971cbdf72254 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_13000.json @@ -0,0 +1 @@ +{"epoch": 0.6158802349819974, "train": {"Classification_binary": 2.795518636703491, "Classification": 0.022668272256851196, "Regression_pt": 0.1530420184135437, "Regression_eta": 0.011242789216339588, "Regression_sin_phi": 0.010020541027188301, "Regression_cos_phi": 0.00870206393301487, "Regression_energy": 0.15580599009990692, "Total": 9.9071626663208, "Total_unweighted": 3.157000311650336}, "train_group": {}, "valid": {"Classification_binary": 2.7088600794474282, "Classification": 0.021755538880825043, "Regression_pt": 0.1398983597755432, "Regression_eta": 0.011072110384702682, "Regression_sin_phi": 0.008528660361965498, "Regression_cos_phi": 0.00850843514005343, "Regression_energy": 0.14342430233955383, "Total": 9.306862513224283, "Total_unweighted": 3.042047486330072, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02229055017232895, "diagnostic/loss/clic_pf/Classification_binary": 2.70841908454895, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008080677129328251, "diagnostic/loss/clic_pf/Regression_energy": 0.14597222208976746, "diagnostic/loss/clic_pf/Regression_eta": 0.010583648458123207, "diagnostic/loss/clic_pf/Regression_pt": 0.1427535116672516, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008259679190814495, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02601570263504982, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19696950912475586, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04156932607293129, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4445120394229889, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005697589833289385, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18923304975032806, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06314694881439209, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4452669620513916, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9645377397537231, "metrics/particle/class_1/pid_accuracy": 0.985986053943634, "metrics/particle/class_2/efficiency": 0.40336039662361145, "metrics/particle/class_2/pid_accuracy": 0.7131674289703369, "metrics/particle/class_3/efficiency": 0.7369112968444824, "metrics/particle/class_3/pid_accuracy": 0.9836204648017883, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.4358707070350647, "metrics/particle/class_5/efficiency": 0.9939637780189514, "metrics/particle/class_5/pid_accuracy": 0.6356275081634521, "metrics/particle/count/bias_mean": -1.5313725471496582, "metrics/particle/count/mae": 3.5535948276519775, "metrics/particle/count/prediction_mean": 68.46797180175781, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05383869633078575, "metrics/particle/event/energy_response_mean": 0.9846785664558411, "metrics/particle/event/met_abs_error": 5.239184856414795, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04235091432929039, "metrics/particle/event/scalar_pt_response_mean": 0.9925200939178467, "metrics/particle/event/vector_pt_closure": 0.044969864189624786, "metrics/particle/matched/delta_eta_abs_mean": 0.013396057300269604, "metrics/particle/matched/delta_phi_abs_mean": 0.010911358520388603, "metrics/particle/matched/delta_r_mean": 0.019322596490383148, "metrics/particle/matched/energy_relative_abs_mean": 0.08459246158599854, "metrics/particle/matched/pid_accuracy": 0.96595698595047, "metrics/particle/matched/pt_relative_abs_mean": 0.07373740524053574, "metrics/particle/matching/duplicate_fraction": 0.028418419882655144, "metrics/particle/matching/efficiency": 0.8192606568336487, "metrics/particle/matching/f1": 0.8283212780952454, "metrics/particle/matching/purity": 0.8375844955444336, "metrics/particle/matching/target_coverage_dr0p05": 0.774759829044342, "metrics/particle/matching/target_coverage_dr0p10": 0.8494663834571838, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7318369150161743, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8581032752990723, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02062046155333519, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08155601471662521, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07359698414802551, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.009796461090445518, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1301727443933487, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002708750544115901, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31885775923728943, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.006460804026573896, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1782708168029785, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8491.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29046937823295593, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1333435773849487, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.271001935005188, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1493794918060303, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12310679256916046, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 39.93239974975586, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014466019347310066, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7745499014854431, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08695586025714874, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.4386348724365234, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10300.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7899339199066162, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.118905544281006, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.429863929748535, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.2065553665161133, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.14599746465682983, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.14096865057945251, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.427318096160889, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 2.4208126068115234, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.919876098632812, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03100624680519104, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09079793840646744, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.085812509059906, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012456022202968597, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14328260719776154, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0199115052819252, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8789461255073547, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07497312873601913, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0572729110717773, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1808.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.708111047744751, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.795785665512085, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.436645984649658, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.2691712379455566, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23525205254554749, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 53.362945556640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07007507979869843, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.687532424926758, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.14463359117507935, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.502411842346191, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2797.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5885416865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.013439178466797, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.081457614898682, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1798770427703857, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.05872407555580139, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.041541069746017456, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.926724910736084, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.0745668411254883, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.653676986694336, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.017858602106571198, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0858423113822937, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07344196736812592, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011258397251367569, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13004493713378906, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000631578965112567, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11643698811531067, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.01575063355267048, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5925809741020203, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19000.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14719827473163605, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5918830633163452, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4472912549972534, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.343887060880661, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08430639654397964, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.484870910644531, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026623073499649763, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24267596006393433, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02908896468579769, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1968343257904053, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21410.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8654459118843079, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.6281055808067322, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.943833827972412, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.575814247131348, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.6336350440979004, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.3592415153980255, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.5759079456329346, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.289834976196289, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.4578275680542, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.019470324739813805, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08490592241287231, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07425025850534439, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010908624157309532, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1309373825788498, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024232908617705107, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2221529185771942, "metrics/particle/momentum/all/accepted/pz_bias": 0.016712931916117668, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0985651016235352, "metrics/particle/momentum/all/accepted/samples": 29299.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23542864620685577, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8510871529579163, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9354659914970398, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8214305639266968, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10812298953533173, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 26.819149017333984, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011649809777736664, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5996054410934448, "metrics/particle/momentum/all/assigned/pz_bias": 0.03228040412068367, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.205897092819214, "metrics/particle/momentum/all/assigned/samples": 34507.0, "metrics/particle/momentum/all/efficiency": 0.8189797401428223, "metrics/particle/momentum/all/event/energy_bias": -6.76045036315918, "metrics/particle/momentum/all/event/p3_error_mean": 14.650199890136719, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.1175456047058105, "metrics/particle/momentum/all/event/px_bias": -0.838356614112854, "metrics/particle/momentum/all/event/py_bias": 0.5417512655258179, "metrics/particle/momentum/all/event/pz_abs_mean": 11.19723892211914, "metrics/particle/momentum/all/event/pz_bias": 2.636080741882324, "metrics/particle/momentum/all/event/pz_residual_rms": 18.94579315185547, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010836613364517689, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04178324714303017, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022206757217645645, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005140065215528011, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06369560211896896, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011074197245761752, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12039818614721298, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.021426253020763397, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.831738293170929, "metrics/particle/momentum/class_1/accepted/samples": 16254.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04756791517138481, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1138935536146164, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11046566814184189, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07967168092727661, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022347362712025642, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.9251174926757812, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00438648508861661, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2397528737783432, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.020571287721395493, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6494219303131104, "metrics/particle/momentum/class_1/assigned/samples": 16870.0, "metrics/particle/momentum/class_1/efficiency": 0.9633141756057739, "metrics/particle/momentum/class_1/event/energy_bias": 1.9930428266525269, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.720947265625, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.968332767486572, "metrics/particle/momentum/class_1/event/px_bias": -0.5922927260398865, "metrics/particle/momentum/class_1/event/py_bias": -0.6109914779663086, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.423886299133301, "metrics/particle/momentum/class_1/event/pz_bias": 1.665113925933838, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.416107177734375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.041064124554395676, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20271487534046173, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2055833786725998, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.039775799959897995, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24569237232208252, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032855939120054245, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6869046688079834, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08833404630422592, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7198660373687744, "metrics/particle/momentum/class_2/accepted/samples": 1187.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9406813979148865, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6308609247207642, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.800567090511322, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.27190542221069336, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36335036158561707, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.3806440830230713, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07786079496145248, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.108365297317505, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2107212245464325, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.294804096221924, "metrics/particle/momentum/class_2/assigned/samples": 2543.0, "metrics/particle/momentum/class_2/efficiency": 0.402919203042984, "metrics/particle/momentum/class_2/event/energy_bias": -11.801990509033203, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.35950756072998, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.2567620277404785, "metrics/particle/momentum/class_2/event/px_bias": -0.21141302585601807, "metrics/particle/momentum/class_2/event/py_bias": 0.07037022709846497, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.784893035888672, "metrics/particle/momentum/class_2/event/pz_bias": 0.7918281555175781, "metrics/particle/momentum/class_2/event/pz_residual_rms": 17.363527297973633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029909593984484673, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13622018694877625, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13643859326839447, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.024731598794460297, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17819252610206604, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001234350260347128, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21324732899665833, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.001768247107975185, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8032477498054504, "metrics/particle/momentum/class_3/accepted/samples": 11342.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33589601516723633, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7650651931762695, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9379124641418457, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7973942756652832, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16563573479652405, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 41.15266036987305, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008865979500114918, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.586051881313324, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.018973151221871376, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.41772198677063, "metrics/particle/momentum/class_3/assigned/samples": 14550.0, "metrics/particle/momentum/class_3/efficiency": 0.7359200716018677, "metrics/particle/momentum/class_3/event/energy_bias": 3.9295976161956787, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.93404769897461, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.238185405731201, "metrics/particle/momentum/class_3/event/px_bias": -0.25499427318573, "metrics/particle/momentum/class_3/event/py_bias": 0.9767576456069946, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.2676262855529785, "metrics/particle/momentum/class_3/event/pz_bias": 0.029058367013931274, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.641857147216797, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01459924504160881, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05532755330204964, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05594473332166672, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.003556338604539633, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10491618514060974, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2767443358898163, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.046374525874853134, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9100264310836792, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.0993598997592926, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.43029236793518066, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4941098093986511, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.4334126114845276, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.82505202293396, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.33483460545539856, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.024578267708420753, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0080262422561646, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.2754371762275696, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9395811557769775, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1880706548690796, "metrics/particle/momentum/class_4/event/px_bias": 0.1124851405620575, "metrics/particle/momentum/class_4/event/py_bias": 0.004505205899477005, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3243787288665771, "metrics/particle/momentum/class_4/event/pz_bias": 0.28311416506767273, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.019832134246826, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008034716360270977, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02401667833328247, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.023503288626670837, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008739694021642208, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07039325684309006, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2109953910112381, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.005339188035577536, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7232155203819275, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033279549330472946, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.029324697330594063, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.028747186064720154, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0033186369109898806, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10090707987546921, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5116209387779236, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29647499322891235, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.134688854217529, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.605663537979126, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0195170640945435, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6731142997741699, "metrics/particle/momentum/class_5/event/px_bias": 0.10785827040672302, "metrics/particle/momentum/class_5/event/py_bias": 0.10110970586538315, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6505470275878906, "metrics/particle/momentum/class_5/event/pz_bias": -0.13303370773792267, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4664554595947266, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4656858444213867, "step_valid_time": 5631.322287321091, "step_total_time": 5632.787973403931, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974802679464805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06811126368695208, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3012, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2882, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2871, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9558872305140962, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9568393094289509, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9563630330180852, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04316069057104914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9522388059701492, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.953187250996016, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9527127924340468, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9558872305140962, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9952789831950002, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0842334334683923, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2127, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2118, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9440745672436751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487065120428189, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.946384872080089, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05129348795718109, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9400798934753661, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.944692239072257, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9423804226918798, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9440745672436751, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9933289935894611, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07860246719657704, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9175097276264591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9196567862714509, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9185820023373588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08034321372854915, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91284046692607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9149765990639626, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9139072847682119, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9175097276264591} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_14000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_14000.json new file mode 100644 index 0000000000000000000000000000000000000000..4cdd3d1435d402c2a37d4b5d3e82dbc9fa5cc572 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_14000.json @@ -0,0 +1 @@ +{"epoch": 0.6632556376729202, "train": {"Classification_binary": 2.6176910400390625, "Classification": 0.018761001527309418, "Regression_pt": 0.1334269642829895, "Regression_eta": 0.013061394914984703, "Regression_sin_phi": 0.008910693228244781, "Regression_cos_phi": 0.0095411017537117, "Regression_energy": 0.13982827961444855, "Total": 9.131603240966797, "Total_unweighted": 2.941220475360751}, "train_group": {}, "valid": {"Classification_binary": 2.6543572743733725, "Classification": 0.021491629381974537, "Regression_pt": 0.13938299814860025, "Regression_eta": 0.011004896213610968, "Regression_sin_phi": 0.008475018044312796, "Regression_cos_phi": 0.008524720867474874, "Regression_energy": 0.14269514878590903, "Total": 9.218188603719076, "Total_unweighted": 2.985931685815255, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02198762819170952, "diagnostic/loss/clic_pf/Classification_binary": 2.666398048400879, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008023105561733246, "diagnostic/loss/clic_pf/Regression_energy": 0.1465492844581604, "diagnostic/loss/clic_pf/Regression_eta": 0.010409878566861153, "diagnostic/loss/clic_pf/Regression_pt": 0.1435738056898117, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008127069100737572, "diagnostic/regression/clic_pf/energy_pred_mean": 0.036519329994916916, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19627906382083893, "diagnostic/regression/clic_pf/energy_residual_mean": 0.052072953432798386, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4449639916419983, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.002107294974848628, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1899450272321701, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07095183432102203, "diagnostic/regression/clic_pf/pt_residual_rms": 0.44599488377571106, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9654719829559326, "metrics/particle/class_1/pid_accuracy": 0.9861231446266174, "metrics/particle/class_2/efficiency": 0.41305819153785706, "metrics/particle/class_2/pid_accuracy": 0.7185367345809937, "metrics/particle/class_3/efficiency": 0.735790491104126, "metrics/particle/class_3/pid_accuracy": 0.9843571782112122, "metrics/particle/class_4/efficiency": 0.9253876209259033, "metrics/particle/class_4/pid_accuracy": 0.41465967893600464, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7433808445930481, "metrics/particle/count/bias_mean": -1.1418300867080688, "metrics/particle/count/mae": 3.4359476566314697, "metrics/particle/count/prediction_mean": 68.85751342773438, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0552905797958374, "metrics/particle/event/energy_response_mean": 1.0123603343963623, "metrics/particle/event/met_abs_error": 6.607929706573486, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04477585107088089, "metrics/particle/event/scalar_pt_response_mean": 1.0149354934692383, "metrics/particle/event/vector_pt_closure": 0.04835903272032738, "metrics/particle/matched/delta_eta_abs_mean": 0.009032880887389183, "metrics/particle/matched/delta_phi_abs_mean": 0.011026117950677872, "metrics/particle/matched/delta_r_mean": 0.01594618335366249, "metrics/particle/matched/energy_relative_abs_mean": 0.08397604525089264, "metrics/particle/matched/pid_accuracy": 0.9667031168937683, "metrics/particle/matched/pt_relative_abs_mean": 0.0743294432759285, "metrics/particle/matching/duplicate_fraction": 0.029425164684653282, "metrics/particle/matching/efficiency": 0.8199516534805298, "metrics/particle/matching/f1": 0.826694130897522, "metrics/particle/matching/purity": 0.8335484862327576, "metrics/particle/matching/target_coverage_dr0p05": 0.7880278825759888, "metrics/particle/matching/target_coverage_dr0p10": 0.851371169090271, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7318462133407593, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8586541414260864, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017496056854724884, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07926440238952637, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07272195816040039, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014721030369400978, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12816044688224792, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00331282545812428, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32168492674827576, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0275596734136343, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2254751920700073, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8452.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29283371567726135, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9377527832984924, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.02103853225708, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9053167104721069, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12783704698085785, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.975902557373047, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014161008410155773, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7341414093971252, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.036936014890670776, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5324699878692627, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10310.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7863057255744934, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.1634860038757324, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.710312366485596, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8700740337371826, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.09372247010469437, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.23355692625045776, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.801651477813721, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.743489146232605, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 12.92132568359375, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026500549167394638, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08688201010227203, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08486387878656387, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01984790712594986, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14192314445972443, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019144143909215927, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.89421546459198, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.03833327442407608, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.098870038986206, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1776.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7095909118652344, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3614463806152344, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.886446475982666, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.738948106765747, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2539285719394684, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 21.73232650756836, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07428571581840515, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.752000093460083, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.039079681038856506, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.628913879394531, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2800.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.578125, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.47779059410095215, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.000481605529785, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2398200035095215, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.06478488445281982, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.016468040645122528, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.8214874267578125, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.6920739412307739, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.2725887298584, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014284908771514893, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08548126369714737, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07411593198776245, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014573677442967892, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1303645372390747, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00042027843301184475, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10593484342098236, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0063202050514519215, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5591281652450562, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19035.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14641696214675903, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7218754291534424, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5294521450996399, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4285375773906708, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08647894114255905, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.724769592285156, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026530136819928885, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2644370198249817, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.02955693006515503, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.677229404449463, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21485.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8670401573181152, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 2.6673173904418945, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.271306991577148, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.9346604347229, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.27079156041145325, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.8671430349349976, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.8597517013549805, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.105647087097168, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 23.137943267822266, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015953760594129562, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08377066999673843, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07436560839414597, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014936335384845734, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13046588003635406, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023920992389321327, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21609123051166534, "metrics/particle/momentum/all/accepted/pz_bias": 0.009744713082909584, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.104504942893982, "metrics/particle/momentum/all/accepted/samples": 29263.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23563343286514282, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.837975800037384, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8667216300964355, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7576239705085754, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11235727369785309, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.244185447692871, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011880329810082912, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6057530641555786, "metrics/particle/momentum/all/assigned/pz_bias": 0.026200834661722183, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.0417160987854, "metrics/particle/momentum/all/assigned/samples": 34595.0, "metrics/particle/momentum/all/efficiency": 0.8179734349250793, "metrics/particle/momentum/all/event/energy_bias": 1.02604079246521, "metrics/particle/momentum/all/event/p3_error_mean": 15.165033340454102, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.360987663269043, "metrics/particle/momentum/all/event/px_bias": -0.2418539822101593, "metrics/particle/momentum/all/event/py_bias": -2.1171679496765137, "metrics/particle/momentum/all/event/pz_abs_mean": 10.710433006286621, "metrics/particle/momentum/all/event/pz_bias": 1.157062292098999, "metrics/particle/momentum/all/event/pz_residual_rms": 27.19224739074707, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007267635315656662, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03950401023030281, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022735051810741425, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.008147520944476128, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0636681467294693, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013540961081162095, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11420346051454544, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.015457008965313435, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8692602515220642, "metrics/particle/momentum/class_1/accepted/samples": 16247.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04503293335437775, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12010261416435242, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10780453681945801, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08169565349817276, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.023885726928710938, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.5404837131500244, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004741583485156298, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2671116888523102, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04934034124016762, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 4.296220302581787, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9628993272781372, "metrics/particle/momentum/class_1/event/energy_bias": 5.620735168457031, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.32862663269043, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.324502944946289, "metrics/particle/momentum/class_1/event/px_bias": -1.3974331617355347, "metrics/particle/momentum/class_1/event/py_bias": -0.9545503258705139, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.23388957977295, "metrics/particle/momentum/class_1/event/pz_bias": 1.4473457336425781, "metrics/particle/momentum/class_1/event/pz_residual_rms": 26.445228576660156, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.039927348494529724, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19805724918842316, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2010495662689209, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03825555369257927, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2414061576128006, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030833333730697632, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.663946270942688, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.10036426037549973, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5687570571899414, "metrics/particle/momentum/class_2/accepted/samples": 1200.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8760305643081665, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6590964794158936, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7937803864479065, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.2777997851371765, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36405888199806213, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.186216115951538, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07552284747362137, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.057246685028076, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11617297679185867, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.037415504455566, "metrics/particle/momentum/class_2/assigned/samples": 2582.0, "metrics/particle/momentum/class_2/efficiency": 0.4073319733142853, "metrics/particle/momentum/class_2/event/energy_bias": -9.572422981262207, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.868741035461426, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.370438098907471, "metrics/particle/momentum/class_2/event/px_bias": 0.2645617425441742, "metrics/particle/momentum/class_2/event/py_bias": -1.4453141689300537, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.097537994384766, "metrics/particle/momentum/class_2/event/pz_bias": 0.8100758790969849, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.289100646972656, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02619396336376667, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13667242228984833, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13609324395656586, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030029891058802605, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17771278321743011, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009731929749250412, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20542018115520477, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008591536432504654, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8277884721755981, "metrics/particle/momentum/class_3/accepted/samples": 11303.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3481852412223816, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6656993627548218, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7313333749771118, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5955711603164673, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17256970703601837, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.44356918334961, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009248475544154644, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.568548858165741, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.011677712202072144, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.895228147506714, "metrics/particle/momentum/class_3/assigned/samples": 14597.0, "metrics/particle/momentum/class_3/efficiency": 0.7333895564079285, "metrics/particle/momentum/class_3/event/energy_bias": 5.469380855560303, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.715289115905762, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.221177101135254, "metrics/particle/momentum/class_3/event/px_bias": 0.7404083609580994, "metrics/particle/momentum/class_3/event/py_bias": 0.276780366897583, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.134945869445801, "metrics/particle/momentum/class_3/event/pz_bias": -1.1722033023834229, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.204301834106445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01113950926810503, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.06907248497009277, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06992071866989136, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.028702568262815475, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11687570065259933, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.3342461884021759, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.04053099453449249, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9403690099716187, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.12470913678407669, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.598039150238037, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.3116679191589355, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.268744945526123, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06077348068356514, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 35.2664909362793, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4549041986465454, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0037124406080693007, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2040632963180542, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.32656654715538025, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.83787202835083, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1656558513641357, "metrics/particle/momentum/class_4/event/px_bias": 0.10763116180896759, "metrics/particle/momentum/class_4/event/py_bias": -0.037321627140045166, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.1956076622009277, "metrics/particle/momentum/class_4/event/pz_bias": 0.3074115216732025, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.496254920959473, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005970432423055172, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.022151511162519455, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021571913734078407, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008835120126605034, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06527368724346161, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.21034692227840424, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01615343987941742, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7409569025039673, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013339100405573845, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03798411041498184, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03692640736699104, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013814523816108704, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1283460110425949, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5039824843406677, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.31261175870895386, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.029073715209961, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.16508550941944122, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0701990127563477, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6937330961227417, "metrics/particle/momentum/class_5/event/px_bias": 0.04297792539000511, "metrics/particle/momentum/class_5/event/py_bias": 0.04323778301477432, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6927019357681274, "metrics/particle/momentum/class_5/event/pz_bias": -0.2355676293373108, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5866549015045166, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.507051944732666, "step_valid_time": 6065.643452167511, "step_total_time": 6067.150504350662, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0041974075089468, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06985787207836935, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2862, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9532338308457712, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9497686715135493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9514980963416653, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05023132848645076, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9492537313432836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9458030403172505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9475252441648734, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9532338308457712, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004593316913973, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07773352198033157, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2229, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2122, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2116, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.941855304039059, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9519964109466128, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9468987059348505, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04800358905338717, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9391921881935198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.949304620906236, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9442213297634985, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.941855304039059, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040084108570009, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07802070651721293, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1175, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.914396887159533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.91796875, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9161793372319688, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08203125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.91284046692607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.91640625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9146198830409357, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.914396887159533} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_15000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_15000.json new file mode 100644 index 0000000000000000000000000000000000000000..23fc9a527ab28865a9f44b7c80e9afa407349e71 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_15000.json @@ -0,0 +1 @@ +{"epoch": 0.7106310403638431, "train": {"Classification_binary": 2.516767740249634, "Classification": 0.021419409662485123, "Regression_pt": 0.11790141463279724, "Regression_eta": 0.010034376755356789, "Regression_sin_phi": 0.006652002688497305, "Regression_cos_phi": 0.006816316861659288, "Regression_energy": 0.11912286281585693, "Total": 8.162703514099121, "Total_unweighted": 2.7987141236662865}, "train_group": {}, "valid": {"Classification_binary": 2.638429323832194, "Classification": 0.02143875261147817, "Regression_pt": 0.1367637018362681, "Regression_eta": 0.010816529393196106, "Regression_sin_phi": 0.00846453383564949, "Regression_cos_phi": 0.008528119573990503, "Regression_energy": 0.14006513357162476, "Total": 9.114643732706705, "Total_unweighted": 2.964506094654401, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.022046120837330818, "diagnostic/loss/clic_pf/Classification_binary": 2.647538423538208, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.0080906692892313, "diagnostic/loss/clic_pf/Regression_energy": 0.14216731488704681, "diagnostic/loss/clic_pf/Regression_eta": 0.0101618692278862, "diagnostic/loss/clic_pf/Regression_pt": 0.13931986689567566, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008042127825319767, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03218730166554451, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19430403411388397, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04774092137813568, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43901386857032776, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0035876668989658356, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18902051448822021, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07243220508098602, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4425337016582489, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.965114176273346, "metrics/particle/class_1/pid_accuracy": 0.9867358803749084, "metrics/particle/class_2/efficiency": 0.41813260316848755, "metrics/particle/class_2/pid_accuracy": 0.7330096960067749, "metrics/particle/class_3/efficiency": 0.742170512676239, "metrics/particle/class_3/pid_accuracy": 0.9829814434051514, "metrics/particle/class_4/efficiency": 0.9050387740135193, "metrics/particle/class_4/pid_accuracy": 0.45396146178245544, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.655804455280304, "metrics/particle/count/bias_mean": -0.34967321157455444, "metrics/particle/count/mae": 3.209803819656372, "metrics/particle/count/prediction_mean": 69.64967346191406, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05272141471505165, "metrics/particle/event/energy_response_mean": 1.0080785751342773, "metrics/particle/event/met_abs_error": 5.784622669219971, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04269910231232643, "metrics/particle/event/scalar_pt_response_mean": 1.013999104499817, "metrics/particle/event/vector_pt_closure": 0.04534044489264488, "metrics/particle/matched/delta_eta_abs_mean": 0.010349780321121216, "metrics/particle/matched/delta_phi_abs_mean": 0.010930962860584259, "metrics/particle/matched/delta_r_mean": 0.016943393275141716, "metrics/particle/matched/energy_relative_abs_mean": 0.08388601243495941, "metrics/particle/matched/pid_accuracy": 0.9671009182929993, "metrics/particle/matched/pt_relative_abs_mean": 0.0747266560792923, "metrics/particle/matching/duplicate_fraction": 0.028827747330069542, "metrics/particle/matching/efficiency": 0.822771430015564, "metrics/particle/matching/f1": 0.8248316049575806, "metrics/particle/matching/purity": 0.8269021511077881, "metrics/particle/matching/target_coverage_dr0p05": 0.7899420261383057, "metrics/particle/matching/target_coverage_dr0p10": 0.8555448651313782, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7331441044807434, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8627998232841492, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02027357928454876, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08023156225681305, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07441312074661255, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016139760613441467, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1300644725561142, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002932207426056266, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31983762979507446, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.027557076886296272, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2523983716964722, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8526.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.299071729183197, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8452698588371277, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0372235774993896, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9191733002662659, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12926149368286133, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.410545349121094, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012484394945204258, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7173255085945129, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06095387041568756, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5199928283691406, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10413.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.793190062046051, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7105351686477661, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.713064193725586, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.902482032775879, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3355667293071747, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3632364273071289, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.835068702697754, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.993219256401062, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.487192153930664, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03131534159183502, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08906169980764389, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0867062509059906, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.026952479034662247, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1423695683479309, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020705092698335648, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9025243520736694, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.020130081102252007, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.88108491897583, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1787.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7542216777801514, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4944252967834473, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.052194118499756, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.894885540008545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2623814642429352, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 52.3322868347168, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07727432250976562, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7619311809539795, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.031548015773296356, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.81683349609375, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2847.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5817057490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2587196826934814, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.434791564941406, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0814622640609741, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.0048013366758823395, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08706333488225937, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.278574466705322, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.991529107093811, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.76657485961914, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.014229416847229004, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08531256020069122, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07451443374156952, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015305032022297382, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13103248178958893, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006284038536250591, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10955286026000977, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008864685893058777, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6215779185295105, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19096.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1482817530632019, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6793570518493652, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5164738297462463, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.414747953414917, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08899383991956711, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.86983585357666, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028244664426892996, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.27514806389808655, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.03588336706161499, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 4.783407688140869, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21597.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8698186874389648, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.3181869983673096, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.430869102478027, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.812652587890625, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.7150216102600098, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.07470208406448364, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.439907073974609, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.6620458364486694, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 30.470962524414062, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01701989211142063, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08406732976436615, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07522588223218918, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016254771500825882, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13147084414958954, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025162366218864918, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21870066225528717, "metrics/particle/momentum/all/accepted/pz_bias": 0.014968346804380417, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.099989652633667, "metrics/particle/momentum/all/accepted/samples": 29409.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24281904101371765, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7954928874969482, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8791489005088806, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7680065035820007, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11518489569425583, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.348033905029297, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011791031807661057, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6103537678718567, "metrics/particle/momentum/all/assigned/pz_bias": 0.03786524012684822, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.58961296081543, "metrics/particle/momentum/all/assigned/samples": 34857.0, "metrics/particle/momentum/all/efficiency": 0.8220545053482056, "metrics/particle/momentum/all/event/energy_bias": 0.34893232583999634, "metrics/particle/momentum/all/event/p3_error_mean": 15.085355758666992, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.136981964111328, "metrics/particle/momentum/all/event/px_bias": -0.37465354800224304, "metrics/particle/momentum/all/event/py_bias": 0.37559762597084045, "metrics/particle/momentum/all/event/pz_abs_mean": 11.110504150390625, "metrics/particle/momentum/all/event/pz_bias": 1.6637362241744995, "metrics/particle/momentum/all/event/pz_residual_rms": 33.75152587890625, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009050958789885044, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03915731981396675, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022446809336543083, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007309738080948591, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06265407055616379, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001476287143304944, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12057116627693176, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.015758773311972618, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8967730402946472, "metrics/particle/momentum/class_1/accepted/samples": 16257.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04898759350180626, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16304071247577667, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.17191125452518463, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.14401741325855255, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.023943578824400902, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.80462121963501, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004682036582380533, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.28772321343421936, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.044725872576236725, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 5.522775650024414, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9634919762611389, "metrics/particle/momentum/class_1/event/energy_bias": 4.460108757019043, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.748895645141602, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.536895275115967, "metrics/particle/momentum/class_1/event/px_bias": -1.6895058155059814, "metrics/particle/momentum/class_1/event/py_bias": -0.9078384637832642, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.710087776184082, "metrics/particle/momentum/class_1/event/pz_bias": 1.4161555767059326, "metrics/particle/momentum/class_1/event/pz_residual_rms": 32.732425689697266, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04100465402007103, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19862879812717438, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2007860690355301, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02637588232755661, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24187348783016205, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028478438034653664, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.606650471687317, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.13937319815158844, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.528587818145752, "metrics/particle/momentum/class_2/accepted/samples": 1229.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9284212589263916, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6834776401519775, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7615973949432373, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.24474117159843445, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37612614035606384, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.7918508052825928, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07657657563686371, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1096532344818115, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07513371855020523, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.505154132843018, "metrics/particle/momentum/class_2/assigned/samples": 2664.0, "metrics/particle/momentum/class_2/efficiency": 0.417175829410553, "metrics/particle/momentum/class_2/event/energy_bias": -7.894840240478516, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.557723045349121, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.0320634841918945, "metrics/particle/momentum/class_2/event/px_bias": 0.7789215445518494, "metrics/particle/momentum/class_2/event/py_bias": -0.4588643014431, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.929620742797852, "metrics/particle/momentum/class_2/event/pz_bias": -0.06722952425479889, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.09263038635254, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.026072485372424126, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1375519335269928, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13829787075519562, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.033664289861917496, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17975978553295135, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013139453949406743, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20902875065803528, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0009428587509319186, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.778852641582489, "metrics/particle/momentum/class_3/accepted/samples": 11416.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3464186191558838, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5132105350494385, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6889342069625854, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.553285002708435, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17467515170574188, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.203590393066406, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008527341298758984, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5356958508491516, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.029277022927999496, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1156320571899414, "metrics/particle/momentum/class_3/assigned/samples": 14776.0, "metrics/particle/momentum/class_3/efficiency": 0.7407215237617493, "metrics/particle/momentum/class_3/event/energy_bias": 4.499363899230957, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.453083992004395, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.195108413696289, "metrics/particle/momentum/class_3/event/px_bias": 0.25598257780075073, "metrics/particle/momentum/class_3/event/py_bias": 1.5011780261993408, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.781667709350586, "metrics/particle/momentum/class_3/event/pz_bias": 0.11674964427947998, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.673327445983887, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012739687226712704, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05537726357579231, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05686841905117035, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.018532993271946907, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09923964738845825, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.25080397725105286, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.002157839247956872, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8402374982833862, "metrics/particle/momentum/class_4/accepted/samples": 328.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11853600293397903, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.18825626373291, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.0825352668762207, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.031972885131836, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.89784049987793, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3612898886203766, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.020242659375071526, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1203981637954712, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9060773253440857, "metrics/particle/momentum/class_4/event/energy_bias": -0.2457558810710907, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.020390748977661, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2577028274536133, "metrics/particle/momentum/class_4/event/px_bias": 0.12578792870044708, "metrics/particle/momentum/class_4/event/py_bias": 0.1095922440290451, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3447989225387573, "metrics/particle/momentum/class_4/event/pz_bias": 0.3887794613838196, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.059116840362549, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00659093726426363, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.017789235338568687, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01772506721317768, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006859091576188803, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.051935382187366486, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1594032347202301, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.006998170167207718, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5362411737442017, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013574968092143536, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.029561713337898254, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.029488958418369293, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00820146407932043, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10941261798143387, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4945199489593506, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3308614194393158, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.030360698699951, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.4699443578720093, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0255799293518066, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6659470200538635, "metrics/particle/momentum/class_5/event/px_bias": 0.15416015684604645, "metrics/particle/momentum/class_5/event/py_bias": 0.13153007626533508, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6621434688568115, "metrics/particle/momentum/class_5/event/pz_bias": -0.19071915745735168, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5471577644348145, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5110931396484375, "step_valid_time": 6500.170145273209, "step_total_time": 6501.681238651276, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005103692795812, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06563372477059382, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3034, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9512195121951219, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9542073069928914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04878048780487805, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9538971807628525, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9479235332893869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9509009753678294, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9572139303482587, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0053999773310816, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08099830239938721, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2121, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2113, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9414114513981359, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9451871657754011, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9432955303535691, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05481283422459893, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378606302707501, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9416221033868093, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.939737602846342, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9414114513981359, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0085909454752573, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07379126589204799, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1291, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.919442292796282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.921583850931677, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08055770720371805, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.91711851278079, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9192546583850931, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9237354085603113} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_16000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_16000.json new file mode 100644 index 0000000000000000000000000000000000000000..2bc551f269a6a3e0d05769b41ed97e72d53ee020 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_16000.json @@ -0,0 +1 @@ +{"epoch": 0.7580064430547659, "train": {"Classification_binary": 2.667080879211426, "Classification": 0.02186983823776245, "Regression_pt": 0.13655421137809753, "Regression_eta": 0.011557036079466343, "Regression_sin_phi": 0.008511139079928398, "Regression_cos_phi": 0.010440698824822903, "Regression_energy": 0.13765767216682434, "Total": 9.291585922241211, "Total_unweighted": 2.9936714749783278}, "train_group": {}, "valid": {"Classification_binary": 2.6495846112569175, "Classification": 0.021456909676392872, "Regression_pt": 0.1379299263159434, "Regression_eta": 0.011103795220454534, "Regression_sin_phi": 0.00837827535967032, "Regression_cos_phi": 0.008492240061362585, "Regression_energy": 0.14115408062934875, "Total": 9.165961583455404, "Total_unweighted": 2.97809983852009, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021750811487436295, "diagnostic/loss/clic_pf/Classification_binary": 2.657200813293457, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008084108121693134, "diagnostic/loss/clic_pf/Regression_energy": 0.14511314034461975, "diagnostic/loss/clic_pf/Regression_eta": 0.010394183918833733, "diagnostic/loss/clic_pf/Regression_pt": 0.14255371689796448, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008311202749609947, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04233560711145401, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.199055477976799, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05788923427462578, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4479403495788574, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.015606780536472797, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19362491369247437, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08445131778717041, "diagnostic/regression/clic_pf/pt_residual_rms": 0.45332059264183044, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.967261016368866, "metrics/particle/class_1/pid_accuracy": 0.9864981770515442, "metrics/particle/class_2/efficiency": 0.43775370717048645, "metrics/particle/class_2/pid_accuracy": 0.7454920411109924, "metrics/particle/class_3/efficiency": 0.7305959463119507, "metrics/particle/class_3/pid_accuracy": 0.9826823472976685, "metrics/particle/class_4/efficiency": 0.9312015771865845, "metrics/particle/class_4/pid_accuracy": 0.44745057821273804, "metrics/particle/class_5/efficiency": 0.9959758520126343, "metrics/particle/class_5/pid_accuracy": 0.6969696879386902, "metrics/particle/count/bias_mean": -0.30588236451148987, "metrics/particle/count/mae": 3.2954249382019043, "metrics/particle/count/prediction_mean": 69.69346618652344, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04906803369522095, "metrics/particle/event/energy_response_mean": 1.0081526041030884, "metrics/particle/event/met_abs_error": 4.778936386108398, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04080149903893471, "metrics/particle/event/scalar_pt_response_mean": 1.0185208320617676, "metrics/particle/event/vector_pt_closure": 0.04320501163601875, "metrics/particle/matched/delta_eta_abs_mean": 0.013307996094226837, "metrics/particle/matched/delta_phi_abs_mean": 0.010148877277970314, "metrics/particle/matched/delta_r_mean": 0.018711866810917854, "metrics/particle/matched/energy_relative_abs_mean": 0.08501474559307098, "metrics/particle/matched/pid_accuracy": 0.9668577909469604, "metrics/particle/matched/pt_relative_abs_mean": 0.07489098608493805, "metrics/particle/matching/duplicate_fraction": 0.03194193169474602, "metrics/particle/matching/efficiency": 0.8206799030303955, "metrics/particle/matching/f1": 0.8224769830703735, "metrics/particle/matching/purity": 0.8242818713188171, "metrics/particle/matching/target_coverage_dr0p05": 0.7749091982841492, "metrics/particle/matching/target_coverage_dr0p10": 0.8542283177375793, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7293345332145691, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8600733876228333, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018993133679032326, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07987060397863388, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07377798855304718, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0193173848092556, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13036714494228363, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0029617343097925186, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.33614951372146606, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0059572854079306126, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2632019519805908, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8441.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2981001138687134, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9820197820663452, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.0660613775253296, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.9636234045028687, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13575269281864166, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.654254913330078, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012192780151963234, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7286837100982666, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0007982100360095501, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.88010573387146, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10416.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7852823734283447, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.2472293376922607, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.384667873382568, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.867187023162842, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.020121311768889427, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.4282121956348419, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.5022382736206055, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.4293186664581299, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.423569679260254, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027688268572092056, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08897054940462112, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09011443704366684, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.028916267678141594, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1451830118894577, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01827242597937584, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9074594378471375, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.008603020571172237, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.1740329265594482, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1806.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.776569128036499, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.692058801651001, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.764639377593994, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.621152400970459, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2649030089378357, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 140.6275634765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07266313582658768, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.89081072807312, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05126378685235977, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.611213684082031, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2835.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.587890625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.9419760704040527, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.429734706878662, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0565540790557861, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.021028947085142136, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09667317569255829, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.270419120788574, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7627153396606445, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.920104026794434, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01787577010691166, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08695673942565918, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07407104969024658, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01531444676220417, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13091489672660828, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000524136470630765, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11700526624917984, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.02505873516201973, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5996650457382202, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19079.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14773114025592804, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7115257382392883, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5461193323135376, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.451284795999527, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08908458799123764, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.091573715209961, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025955967139452696, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.25934234261512756, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01217240747064352, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.028243064880371, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21575.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8690443634986877, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.8103219270706177, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.732545375823975, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.449694633483887, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4975598454475403, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.9410269856452942, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.452340602874756, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3763372302055359, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.141946792602539, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018801676109433174, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08504112809896469, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07497470825910568, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017304275184869766, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.131681889295578, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002318761544302106, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22876128554344177, "metrics/particle/momentum/all/accepted/pz_bias": -0.015117889270186424, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1461504697799683, "metrics/particle/momentum/all/accepted/samples": 29326.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.24389487504959106, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9536517262458801, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.126439094543457, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.0253701210021973, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1173548474907875, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 40.8265495300293, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011169815436005592, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6139300465583801, "metrics/particle/momentum/all/assigned/pz_bias": -0.01147528737783432, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.542490243911743, "metrics/particle/momentum/all/assigned/samples": 34826.0, "metrics/particle/momentum/all/efficiency": 0.8197344541549683, "metrics/particle/momentum/all/event/energy_bias": 0.11557500064373016, "metrics/particle/momentum/all/event/p3_error_mean": 13.9026460647583, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.97205114364624, "metrics/particle/momentum/all/event/px_bias": -0.5387101173400879, "metrics/particle/momentum/all/event/py_bias": 1.4659123420715332, "metrics/particle/momentum/all/event/pz_abs_mean": 10.465761184692383, "metrics/particle/momentum/all/event/pz_bias": -1.5683711767196655, "metrics/particle/momentum/all/event/pz_residual_rms": 18.846940994262695, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00943465530872345, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04151309281587601, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021772244945168495, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.001722894492559135, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06190931424498558, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013497760519385338, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12342984229326248, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.015667734667658806, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8625636696815491, "metrics/particle/momentum/class_1/accepted/samples": 16299.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.044197723269462585, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13566343486309052, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11201430857181549, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.0830121785402298, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021281640976667404, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.8337242603302002, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004208903759717941, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2430274933576584, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.016609463840723038, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.53145694732666, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.965981125831604, "metrics/particle/momentum/class_1/event/energy_bias": 3.477908134460449, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.649286270141602, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.122899055480957, "metrics/particle/momentum/class_1/event/px_bias": -0.6985901594161987, "metrics/particle/momentum/class_1/event/py_bias": -0.5522685050964355, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.231391429901123, "metrics/particle/momentum/class_1/event/pz_bias": -0.47432076930999756, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.017534255981445, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04202861711382866, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19600796699523926, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19823193550109863, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04050849378108978, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23913836479187012, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.025502318516373634, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6081037521362305, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.001195210381411016, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5428829193115234, "metrics/particle/momentum/class_2/accepted/samples": 1294.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9033718109130859, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6950832009315491, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8327027559280396, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.35760077834129333, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3493253290653229, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.4659576416015625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07383807748556137, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1257104873657227, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10136247426271439, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.838008880615234, "metrics/particle/momentum/class_2/assigned/samples": 2668.0, "metrics/particle/momentum/class_2/efficiency": 0.43923965096473694, "metrics/particle/momentum/class_2/event/energy_bias": -7.555772304534912, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.158635139465332, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.807558536529541, "metrics/particle/momentum/class_2/event/px_bias": -0.45738112926483154, "metrics/particle/momentum/class_2/event/py_bias": 0.13814884424209595, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.829849243164062, "metrics/particle/momentum/class_2/event/pz_bias": -1.0595097541809082, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.785318374633789, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030085299164056778, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13753466308116913, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13967786729335785, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04728173837065697, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18121761083602905, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011588518973439932, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22208985686302185, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01355368085205555, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9348475337028503, "metrics/particle/momentum/class_3/accepted/samples": 11218.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3592662513256073, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.9147958755493164, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.332808017730713, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.214890956878662, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18786029517650604, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 62.52842330932617, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008138352073729038, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.591452419757843, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.020044097676873207, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.309682846069336, "metrics/particle/momentum/class_3/assigned/samples": 14745.0, "metrics/particle/momentum/class_3/efficiency": 0.7278743982315063, "metrics/particle/momentum/class_3/event/energy_bias": 5.570699214935303, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.766225814819336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.294569969177246, "metrics/particle/momentum/class_3/event/px_bias": 0.4424015283584595, "metrics/particle/momentum/class_3/event/py_bias": 1.6222507953643799, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.152607440948486, "metrics/particle/momentum/class_3/event/pz_bias": -0.01398875005543232, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.174283981323242, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012891198508441448, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0512537881731987, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05061326548457146, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0009773700730875134, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09493745863437653, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2604256272315979, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03269827738404274, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8734950423240662, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09599197655916214, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.294339418411255, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.979001760482788, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9239420890808105, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06906077265739441, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 29.234460830688477, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.35404935479164124, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.03537414222955704, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0459606647491455, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.9153937101364136, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6154310703277588, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1083297729492188, "metrics/particle/momentum/class_4/event/px_bias": 0.16225887835025787, "metrics/particle/momentum/class_4/event/py_bias": 0.22645457088947296, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9603184461593628, "metrics/particle/momentum/class_4/event/pz_bias": 0.15001757442951202, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.274296522140503, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007816949859261513, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020317932590842247, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.019442494958639145, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005904408637434244, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05939258635044098, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.20772358775138855, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.14673475921154022, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7308975458145142, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03289128467440605, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.025641271844506264, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.024701248854398727, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0005063848220743239, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09352957457304001, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5086358189582825, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4479821026325226, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.13960075378418, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.46186578273773193, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9814945459365845, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6345264315605164, "metrics/particle/momentum/class_5/event/px_bias": 0.012600772082805634, "metrics/particle/momentum/class_5/event/py_bias": 0.03132665529847145, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6463687419891357, "metrics/particle/momentum/class_5/event/pz_bias": -0.17056941986083984, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.476499319076538, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.55350923538208, "step_valid_time": 6932.28083729744, "step_total_time": 6933.83434677124, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0056584272624263, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07265509655357161, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2879, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952820851204223, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.955342375124049, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04717914879577697, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9548922056384743, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9498515341471462, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9523652001323188, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578772802653399, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.011841526548046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0832325831821582, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2124, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9449622725255216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9470640569395018, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9460119973339258, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052935943060498224, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9427430093209055, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9448398576512456, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9437902688291491, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9449622725255216, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0107292569484347, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07707380853124379, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1161, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1157, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9035019455252918, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9084507042253521, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.905969566913773, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09154929577464789, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9003891050583658, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.905320813771518, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9028482247366367, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9035019455252918} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_17000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_17000.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e6b04ef2304682f3a7b09f236a0604cf28766b --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_17000.json @@ -0,0 +1 @@ +{"epoch": 0.8053818457456888, "train": {"Classification_binary": 2.560429811477661, "Classification": 0.02032049372792244, "Regression_pt": 0.15717121958732605, "Regression_eta": 0.011197781190276146, "Regression_sin_phi": 0.008846734650433064, "Regression_cos_phi": 0.0091912392526865, "Regression_energy": 0.15336550772190094, "Total": 9.565530776977539, "Total_unweighted": 2.9205227876082063}, "train_group": {}, "valid": {"Classification_binary": 2.604625701904297, "Classification": 0.020963927110036213, "Regression_pt": 0.13485238949457803, "Regression_eta": 0.010900415480136871, "Regression_sin_phi": 0.008539892733097076, "Regression_cos_phi": 0.008503216629227003, "Regression_energy": 0.1385412315527598, "Total": 9.017878850301107, "Total_unweighted": 2.926926774904132, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.021454323083162308, "diagnostic/loss/clic_pf/Classification_binary": 2.615793228149414, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00815821997821331, "diagnostic/loss/clic_pf/Regression_energy": 0.14213228225708008, "diagnostic/loss/clic_pf/Regression_eta": 0.010219007730484009, "diagnostic/loss/clic_pf/Regression_pt": 0.1388269066810608, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008047393523156643, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03025788813829422, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19389472901821136, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04581150785088539, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43779051303863525, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.003194313496351242, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18713875114917755, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07203885167837143, "diagnostic/regression/clic_pf/pt_residual_rms": 0.44083544611930847, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9679368734359741, "metrics/particle/class_1/pid_accuracy": 0.9874112010002136, "metrics/particle/class_2/efficiency": 0.41114118695259094, "metrics/particle/class_2/pid_accuracy": 0.7366977334022522, "metrics/particle/class_3/efficiency": 0.7380321025848389, "metrics/particle/class_3/pid_accuracy": 0.9840250015258789, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.43795621395111084, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7195122241973877, "metrics/particle/count/bias_mean": -0.8039215803146362, "metrics/particle/count/mae": 3.3032679557800293, "metrics/particle/count/prediction_mean": 69.19542694091797, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05288610979914665, "metrics/particle/event/energy_response_mean": 0.988542377948761, "metrics/particle/event/met_abs_error": 5.456933498382568, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03990030661225319, "metrics/particle/event/scalar_pt_response_mean": 1.0012699365615845, "metrics/particle/event/vector_pt_closure": 0.04450010880827904, "metrics/particle/matched/delta_eta_abs_mean": 0.011817887425422668, "metrics/particle/matched/delta_phi_abs_mean": 0.011520739644765854, "metrics/particle/matched/delta_r_mean": 0.018476348370313644, "metrics/particle/matched/energy_relative_abs_mean": 0.08417700976133347, "metrics/particle/matched/pid_accuracy": 0.9682274460792542, "metrics/particle/matched/pt_relative_abs_mean": 0.07385920733213425, "metrics/particle/matching/duplicate_fraction": 0.027949634939432144, "metrics/particle/matching/efficiency": 0.8219684362411499, "metrics/particle/matching/f1": 0.8267157673835754, "metrics/particle/matching/purity": 0.8315181732177734, "metrics/particle/matching/target_coverage_dr0p05": 0.7756935358047485, "metrics/particle/matching/target_coverage_dr0p10": 0.8537054657936096, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7319956421852112, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8618100881576538, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.021336570382118225, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07928302884101868, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07273361086845398, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016417408362030983, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12972970306873322, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0036504946183413267, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3150252401828766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.024292754009366035, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2263730764389038, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8492.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.29332974553108215, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8116523027420044, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8245682120323181, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7073737382888794, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12460125982761383, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.012263298034668, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013629772700369358, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.701962947845459, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03646229952573776, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5480117797851562, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10345.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7900269627571106, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -3.4204349517822266, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.600499629974365, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.031203269958496, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5142897963523865, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.23061878979206085, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.627125263214111, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.07710293680429459, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.94034481048584, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028553621843457222, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0891040712594986, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08681869506835938, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02294779010117054, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14380010962486267, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020798200741410255, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8885371088981628, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1273801326751709, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9742271900177, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1779.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7655132412910461, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1130820512771606, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.073650598526001, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9039208889007568, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2588192820549011, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 17.647050857543945, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07559395581483841, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.728562593460083, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.18261949717998505, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.8766069412231445, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2778.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5791015625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.763089179992676, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.768899917602539, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0532724857330322, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.007889065891504288, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.03456244245171547, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.637486934661865, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.21461445093154907, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.552440643310547, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016255658119916916, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08661720156669617, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07385954260826111, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014766184613108635, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13182808458805084, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005227391375228763, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10849151015281677, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.013621746562421322, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.550338625907898, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19130.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14544180035591125, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5569256544113159, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.440001517534256, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3389326333999634, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08515668660402298, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.799476146697998, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002225106582045555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24522800743579865, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0035823904909193516, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3744826316833496, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21572.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8713673949241638, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0832055807113647, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.978203773498535, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.695282936096191, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.872285783290863, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.29153338074684143, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.555290460586548, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.056257009506225586, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.795699119567871, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01846732385456562, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08464933186769485, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07431846857070923, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015738166868686676, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13198402523994446, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026529710739851, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21534451842308044, "metrics/particle/momentum/all/accepted/pz_bias": 0.0058610024861991405, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.080151081085205, "metrics/particle/momentum/all/accepted/samples": 29401.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2391861230134964, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6774084568023682, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6854726076126099, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5740979909896851, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1108228862285614, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.641902446746826, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011500216089189053, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5802510976791382, "metrics/particle/momentum/all/assigned/pz_bias": 0.023266758769750595, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2262959480285645, "metrics/particle/momentum/all/assigned/samples": 34695.0, "metrics/particle/momentum/all/efficiency": 0.8218308687210083, "metrics/particle/momentum/all/event/energy_bias": -6.100318431854248, "metrics/particle/momentum/all/event/p3_error_mean": 14.28979206085205, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.133228302001953, "metrics/particle/momentum/all/event/px_bias": 1.3944646120071411, "metrics/particle/momentum/all/event/py_bias": -0.09547702968120575, "metrics/particle/momentum/all/event/pz_abs_mean": 10.814493179321289, "metrics/particle/momentum/all/event/pz_bias": -0.34797435998916626, "metrics/particle/momentum/all/event/pz_residual_rms": 20.431604385375977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008560261689126492, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04060337319970131, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02057051658630371, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0010133583564311266, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06125587970018387, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014106102753430605, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10883679986000061, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00323888729326427, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.782843291759491, "metrics/particle/momentum/class_1/accepted/samples": 16305.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04447666555643082, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13840094208717346, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10341408848762512, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07073776423931122, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020862968638539314, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7702138423919678, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.00486012315377593, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.25860992074012756, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0022160999942570925, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.981900215148926, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.966336727142334, "metrics/particle/momentum/class_1/event/energy_bias": 1.005534052848816, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.665193557739258, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.182051181793213, "metrics/particle/momentum/class_1/event/px_bias": -0.8050692081451416, "metrics/particle/momentum/class_1/event/py_bias": -0.1717267632484436, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.294312000274658, "metrics/particle/momentum/class_1/event/pz_bias": -0.28614795207977295, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.247121810913086, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04112129658460617, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20144665241241455, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20244240760803223, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.030131520703434944, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24439497292041779, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03294892981648445, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6757996082305908, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.14146576821804047, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.741365432739258, "metrics/particle/momentum/class_2/accepted/samples": 1214.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9402572512626648, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6962258815765381, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7686715722084045, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.2538360357284546, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36856263875961304, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.166559934616089, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08070714771747589, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1575369834899902, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.21710924804210663, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.614654541015625, "metrics/particle/momentum/class_2/assigned/samples": 2602.0, "metrics/particle/momentum/class_2/efficiency": 0.4120841920375824, "metrics/particle/momentum/class_2/event/energy_bias": -10.56686019897461, "metrics/particle/momentum/class_2/event/p3_error_mean": 14.053353309631348, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.101284503936768, "metrics/particle/momentum/class_2/event/px_bias": 1.2784981727600098, "metrics/particle/momentum/class_2/event/py_bias": -0.7608139514923096, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.557289123535156, "metrics/particle/momentum/class_2/event/pz_bias": -0.6110670566558838, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.34060287475586, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030654609203338623, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13733379542827606, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13923661410808563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04226004332304001, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1815268099308014, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013192612677812576, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21069768071174622, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007571215275675058, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.782507598400116, "metrics/particle/momentum/class_3/accepted/samples": 11370.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3445320129394531, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3085871934890747, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.353645920753479, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.220930814743042, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1710158735513687, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.505548477172852, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00722218444570899, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4989696443080902, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02627411298453808, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.9372321367263794, "metrics/particle/momentum/class_3/assigned/samples": 14677.0, "metrics/particle/momentum/class_3/efficiency": 0.7377368211746216, "metrics/particle/momentum/class_3/event/energy_bias": 4.3228254318237305, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.280981063842773, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.025362968444824, "metrics/particle/momentum/class_3/event/px_bias": 0.5606210231781006, "metrics/particle/momentum/class_3/event/py_bias": 0.8194578289985657, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.715370178222656, "metrics/particle/momentum/class_3/event/pz_bias": 0.5098999738693237, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.940518379211426, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011255468241870403, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05095234513282776, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05175068601965904, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004388697445392609, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09741102159023285, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26626354455947876, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.023866962641477585, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8542346954345703, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10541138052940369, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.3815802335739136, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4417228102684021, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3823500871658325, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.4227843284606934, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3282853066921234, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.045269548892974854, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9865533709526062, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.5100679993629456, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.9375278949737549, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2328914403915405, "metrics/particle/momentum/class_4/event/px_bias": 0.2742435336112976, "metrics/particle/momentum/class_4/event/py_bias": -0.0533086359500885, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.274121642112732, "metrics/particle/momentum/class_4/event/pz_bias": 0.2778528034687042, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.827641010284424, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006538993213325739, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020827218890190125, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.019645966589450836, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004562124144285917, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05994867905974388, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.21252118051052094, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.13825176656246185, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7914056181907654, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.037075649946928024, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06449329107999802, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.056174177676439285, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.034873031079769135, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4232085645198822, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6066868305206299, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.49189916253089905, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 5.142842769622803, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.3517502248287201, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9932054281234741, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6334279775619507, "metrics/particle/momentum/class_5/event/px_bias": 0.08617113530635834, "metrics/particle/momentum/class_5/event/py_bias": 0.07091453671455383, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6588588953018188, "metrics/particle/momentum/class_5/event/pz_bias": -0.23851211369037628, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.5775129795074463, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4093413352966309, "step_valid_time": 7366.392355918884, "step_total_time": 7367.8016974925995, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9987789669472669, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0654624460627935, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2878, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2870, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9545605306799336, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9510905485789821, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952822380400596, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.048909451421017845, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9519071310116086, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9484467944481163, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9501738122827347, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9545605306799336, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9981010971858724, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07743429840275817, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2235, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2112, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2106, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9374167776298269, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9449664429530201, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9411764705882353, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05503355704697987, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347536617842876, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9422818791946309, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9385026737967914, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9374167776298269, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031905246576047, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0771369865336694, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1183, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9251753702260327, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9244548286604362, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07482462977396727, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9206225680933852, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9220576773187841, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9213395638629284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9237354085603113} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_18000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_18000.json new file mode 100644 index 0000000000000000000000000000000000000000..ad92af7d2d12023ba95f9b653ebdc2445bb0a0c4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_18000.json @@ -0,0 +1 @@ +{"epoch": 0.8527572484366117, "train": {"Classification_binary": 2.434931993484497, "Classification": 0.019258061423897743, "Regression_pt": 0.1265825629234314, "Regression_eta": 0.008281128481030464, "Regression_sin_phi": 0.008206627331674099, "Regression_cos_phi": 0.008770942687988281, "Regression_energy": 0.12629622220993042, "Total": 8.353121757507324, "Total_unweighted": 2.7323275385424495}, "train_group": {}, "valid": {"Classification_binary": 2.5874937375386557, "Classification": 0.021121447285016377, "Regression_pt": 0.13318098584810892, "Regression_eta": 0.01083044707775116, "Regression_sin_phi": 0.00831160694360733, "Regression_cos_phi": 0.008412988856434822, "Regression_energy": 0.13659884532292685, "Total": 8.932854334513346, "Total_unweighted": 2.9059500588725014, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.0215722918510437, "diagnostic/loss/clic_pf/Classification_binary": 2.5962741374969482, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.0079976636916399, "diagnostic/loss/clic_pf/Regression_energy": 0.13901826739311218, "diagnostic/loss/clic_pf/Regression_eta": 0.01013268530368805, "diagnostic/loss/clic_pf/Regression_pt": 0.1360328495502472, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007895069196820259, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03608443960547447, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19115877151489258, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05163806304335594, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4314366579055786, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0015697346534579992, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1844685673713684, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0672748014330864, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4327354431152344, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9677976965904236, "metrics/particle/class_1/pid_accuracy": 0.9837738871574402, "metrics/particle/class_2/efficiency": 0.43809202313423157, "metrics/particle/class_2/pid_accuracy": 0.7266409397125244, "metrics/particle/class_3/efficiency": 0.7536803483963013, "metrics/particle/class_3/pid_accuracy": 0.9842137098312378, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.5359749794006348, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7357723712921143, "metrics/particle/count/bias_mean": 0.3633987009525299, "metrics/particle/count/mae": 3.22352933883667, "metrics/particle/count/prediction_mean": 70.36274719238281, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.049580492079257965, "metrics/particle/event/energy_response_mean": 1.0158371925354004, "metrics/particle/event/met_abs_error": 4.447360992431641, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038538072258234024, "metrics/particle/event/scalar_pt_response_mean": 1.0127842426300049, "metrics/particle/event/vector_pt_closure": 0.040683336555957794, "metrics/particle/matched/delta_eta_abs_mean": 0.01098488550633192, "metrics/particle/matched/delta_phi_abs_mean": 0.012360591441392899, "metrics/particle/matched/delta_r_mean": 0.01838349737226963, "metrics/particle/matched/energy_relative_abs_mean": 0.08469230681657791, "metrics/particle/matched/pid_accuracy": 0.9665243029594421, "metrics/particle/matched/pt_relative_abs_mean": 0.07464201003313065, "metrics/particle/matching/duplicate_fraction": 0.029288003221154213, "metrics/particle/matching/efficiency": 0.8309134244918823, "metrics/particle/matching/f1": 0.8287622332572937, "metrics/particle/matching/purity": 0.8266220688819885, "metrics/particle/matching/target_coverage_dr0p05": 0.7984761595726013, "metrics/particle/matching/target_coverage_dr0p10": 0.8638643026351929, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7411087155342102, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8703815937042236, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.020780468359589577, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08156923949718475, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07426196336746216, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012395202182233334, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13051821291446686, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0024176836013793945, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3204507827758789, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02501930296421051, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1608121395111084, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8686.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.279209166765213, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8942781090736389, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8827224373817444, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.767781138420105, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12238294631242752, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.725445747375488, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013894176110625267, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7282015681266785, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.06165175884962082, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.836552381515503, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10508.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8080751895904541, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.843262195587158, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.9803466796875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.062161922454834, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.052025455981492996, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.2780577838420868, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.028512954711914, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4358879327774048, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.454007148742676, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.027438824996352196, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.092352494597435, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08954938501119614, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.016022201627492905, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14525410532951355, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01879699155688286, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.895721971988678, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11762949079275131, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.806900978088379, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1862.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7502395510673523, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6722829341888428, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 59045.390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 59045.22265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2375824898481369, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3168002.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07537339627742767, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.730876922607422, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10043933987617493, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.642317771911621, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2879.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6061198115348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.47940635681152344, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.6287841796875, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1194772720336914, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09948600828647614, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.09658000618219376, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.460289478302002, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.17143100500106812, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.138031005859375, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.016463981941342354, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08591003715991974, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07435483485460281, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01589467003941536, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13113050162792206, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0003653826133813709, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10962281376123428, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.004057413432747126, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.519920825958252, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19158.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14786043763160706, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.659218430519104, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.48204827308654785, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.38218680024147034, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08795269578695297, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.790131092071533, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002910199575126171, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24915723502635956, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.006512822583317757, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2722752094268799, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21648.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8726428151130676, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.17997723817825317, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.322968482971191, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.179284572601318, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.39792758226394653, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5411366820335388, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1654553413391113, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.28219518065452576, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.378359317779541, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.018414031714200974, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08504460752010345, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0752800926566124, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014879422262310982, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1318826824426651, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0021207837853580713, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22054210305213928, "metrics/particle/momentum/all/accepted/pz_bias": 0.017305448651313782, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0306212902069092, "metrics/particle/momentum/all/accepted/samples": 29706.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23675616085529327, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8129681348800659, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4852.615234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4852.50634765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11057513952255249, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 908145.3125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01215926930308342, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5967715978622437, "metrics/particle/momentum/all/assigned/pz_bias": 0.030768979340791702, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8658061027526855, "metrics/particle/momentum/all/assigned/samples": 35035.0, "metrics/particle/momentum/all/efficiency": 0.8303564190864563, "metrics/particle/momentum/all/event/energy_bias": 2.543833017349243, "metrics/particle/momentum/all/event/p3_error_mean": 13.535406112670898, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.321399688720703, "metrics/particle/momentum/all/event/px_bias": -0.5494390726089478, "metrics/particle/momentum/all/event/py_bias": -0.3596588969230652, "metrics/particle/momentum/all/event/pz_abs_mean": 10.480713844299316, "metrics/particle/momentum/all/event/pz_bias": 0.8895142078399658, "metrics/particle/momentum/all/event/pz_residual_rms": 17.339691162109375, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009165347553789616, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03951708599925041, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022783668711781502, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00708420854061842, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06392882019281387, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.000981655321083963, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12017262727022171, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.02083793468773365, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7577386498451233, "metrics/particle/momentum/class_1/accepted/samples": 16299.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04567211866378784, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12596359848976135, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09600862115621567, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06970418244600296, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02056661993265152, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.462890386581421, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0043266951106488705, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23689985275268555, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0011036392534151673, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7956568002700806, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.965981125831604, "metrics/particle/momentum/class_1/event/energy_bias": 3.4030189514160156, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.09593391418457, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.492309093475342, "metrics/particle/momentum/class_1/event/px_bias": -0.3514379858970642, "metrics/particle/momentum/class_1/event/py_bias": 0.08550812304019928, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.02533483505249, "metrics/particle/momentum/class_1/event/pz_bias": 0.1732037216424942, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.237985610961914, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04086147993803024, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19421027600765228, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1968054473400116, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03728820011019707, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23869746923446655, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028593508526682854, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5351569652557373, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07336166501045227, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.408175468444824, "metrics/particle/momentum/class_2/accepted/samples": 1294.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9361763596534729, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6793397068977356, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 62311.39453125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 62310.88671875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36620235443115234, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3254499.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08247800916433334, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.116563558578491, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.23264798521995544, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.180612564086914, "metrics/particle/momentum/class_2/assigned/samples": 2728.0, "metrics/particle/momentum/class_2/efficiency": 0.43923965096473694, "metrics/particle/momentum/class_2/event/energy_bias": -7.901832580566406, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.896835327148438, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.0560688972473145, "metrics/particle/momentum/class_2/event/px_bias": 0.11644285917282104, "metrics/particle/momentum/class_2/event/py_bias": -1.0250835418701172, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.370471954345703, "metrics/particle/momentum/class_2/event/pz_bias": 1.0545672178268433, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.638219833374023, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.029242243617773056, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13868261873722076, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13692250847816467, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.031966280192136765, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1791146695613861, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008622176246717572, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21446096897125244, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007293939124792814, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7672205567359924, "metrics/particle/momentum/class_3/accepted/samples": 11598.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33051347732543945, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6196726560592651, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6126207113265991, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.479432463645935, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1684238761663437, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.955183982849121, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008528641425073147, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5511807799339294, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.032450150698423386, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.447772741317749, "metrics/particle/momentum/class_3/assigned/samples": 14891.0, "metrics/particle/momentum/class_3/efficiency": 0.7525305151939392, "metrics/particle/momentum/class_3/event/energy_bias": 7.43734073638916, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.72389030456543, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.17940616607666, "metrics/particle/momentum/class_3/event/px_bias": -0.43568921089172363, "metrics/particle/momentum/class_3/event/py_bias": 0.5658796429634094, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.140143394470215, "metrics/particle/momentum/class_3/event/pz_bias": -0.28515195846557617, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.901144027709961, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012851486913859844, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05430688336491585, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05437176302075386, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.00790715403854847, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10274630039930344, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24760591983795166, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.006661138031631708, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8537160158157349, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11743921041488647, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0483288764953613, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7458509802818298, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6914333701133728, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.967031002044678, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3041066825389862, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0028239956591278315, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.975170910358429, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.17843955755233765, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5775643587112427, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.0063073635101318, "metrics/particle/momentum/class_4/event/px_bias": 0.14929717779159546, "metrics/particle/momentum/class_4/event/py_bias": -0.00904674269258976, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0288121700286865, "metrics/particle/momentum/class_4/event/pz_bias": 0.06291116029024124, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.028651714324951, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00716450996696949, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02391248755156994, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.022284913808107376, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007776380050927401, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0629664957523346, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19983333349227905, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.01586359553039074, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7189664840698242, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0334838442504406, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03187011554837227, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.029583845287561417, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00448819762095809, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10016676038503647, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5011754035949707, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.31585749983787537, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.134578227996826, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.21625478565692902, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.0022932291030884, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6113264560699463, "metrics/particle/momentum/class_5/event/px_bias": -0.02805187553167343, "metrics/particle/momentum/class_5/event/py_bias": 0.023083649575710297, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6732239127159119, "metrics/particle/momentum/class_5/event/pz_bias": -0.1160159707069397, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.704827308654785, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5773262977600098, "step_valid_time": 7800.876488208771, "step_total_time": 7802.453814983368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023436445502942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0636917133974465, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2878, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2873, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9545605306799336, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501485638824695, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9523494374586366, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04985143611753054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9529021558872305, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9484978540772532, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9506949040370616, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9545605306799336, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0057100726349264, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.075345202537067, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2127, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9467376830892144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547896150402865, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9507466012926231, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045210384959713516, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9440745672436751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9521038495971352, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9480722086026299, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9467376830892144, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0071212892486434, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07544959897717196, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1179, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1174, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9175097276264591, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9196567862714509, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9185820023373588, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08034321372854915, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9136186770428015, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9157566302652106, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9146864043630697, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9175097276264591} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_19000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_19000.json new file mode 100644 index 0000000000000000000000000000000000000000..2bd7ae1a59445e0c16d83a543d2a96717ca1f076 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_19000.json @@ -0,0 +1 @@ +{"epoch": 0.9001326511275346, "train": {"Classification_binary": 2.496398687362671, "Classification": 0.019073164090514183, "Regression_pt": 0.12503103911876678, "Regression_eta": 0.009116808883845806, "Regression_sin_phi": 0.007244538981467485, "Regression_cos_phi": 0.00879044272005558, "Regression_energy": 0.12509691715240479, "Total": 8.354374885559082, "Total_unweighted": 2.7907515983097255}, "train_group": {}, "valid": {"Classification_binary": 2.5790751775105796, "Classification": 0.02058471366763115, "Regression_pt": 0.13300199309984842, "Regression_eta": 0.010662811497847239, "Regression_sin_phi": 0.008358574782808622, "Regression_cos_phi": 0.00837914397319158, "Regression_energy": 0.13645997643470764, "Total": 8.889521280924479, "Total_unweighted": 2.8965223909666142, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020960045978426933, "diagnostic/loss/clic_pf/Classification_binary": 2.5887699127197266, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007979784160852432, "diagnostic/loss/clic_pf/Regression_energy": 0.14065977931022644, "diagnostic/loss/clic_pf/Regression_eta": 0.00994597002863884, "diagnostic/loss/clic_pf/Regression_pt": 0.1374611109495163, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007947754114866257, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04255932942032814, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19364097714424133, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05811294913291931, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43613487482070923, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.00958454329520464, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18705379962921143, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07842908054590225, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4385030269622803, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.970461368560791, "metrics/particle/class_1/pid_accuracy": 0.9863173365592957, "metrics/particle/class_2/efficiency": 0.4362877905368805, "metrics/particle/class_2/pid_accuracy": 0.7397260069847107, "metrics/particle/class_3/efficiency": 0.7485504746437073, "metrics/particle/class_3/pid_accuracy": 0.9851709008216858, "metrics/particle/class_4/efficiency": 0.940891444683075, "metrics/particle/class_4/pid_accuracy": 0.4706488251686096, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.707317054271698, "metrics/particle/count/bias_mean": 0.9137254953384399, "metrics/particle/count/mae": 3.1960785388946533, "metrics/particle/count/prediction_mean": 70.91307067871094, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.053785961121320724, "metrics/particle/event/energy_response_mean": 1.0296151638031006, "metrics/particle/event/met_abs_error": 5.089086532592773, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04472396522760391, "metrics/particle/event/scalar_pt_response_mean": 1.0294545888900757, "metrics/particle/event/vector_pt_closure": 0.04203295335173607, "metrics/particle/matched/delta_eta_abs_mean": 0.010835620574653149, "metrics/particle/matched/delta_phi_abs_mean": 0.010526646859943867, "metrics/particle/matched/delta_r_mean": 0.01704992540180683, "metrics/particle/matched/energy_relative_abs_mean": 0.08495881408452988, "metrics/particle/matched/pid_accuracy": 0.9679574966430664, "metrics/particle/matched/pt_relative_abs_mean": 0.07480227202177048, "metrics/particle/matching/duplicate_fraction": 0.03186263144016266, "metrics/particle/matching/efficiency": 0.8299050331115723, "metrics/particle/matching/f1": 0.8245236277580261, "metrics/particle/matching/purity": 0.8192116022109985, "metrics/particle/matching/target_coverage_dr0p05": 0.8059272170066833, "metrics/particle/matching/target_coverage_dr0p10": 0.8670482635498047, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7368042469024658, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8681313395500183, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.019293000921607018, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07957939803600311, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07280140370130539, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018033094704151154, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1288340985774994, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028958646580576897, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3213813602924347, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.022670913487672806, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2050176858901978, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8633.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28691399097442627, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0158575773239136, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.037265658378601, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.92659991979599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13183066248893738, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.181933403015137, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.014205890707671642, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7851073145866394, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.006218955852091312, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.039406776428223, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10559.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8031444549560547, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.661468505859375, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.785593509674072, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.90132212638855, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.5328206419944763, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.15530946850776672, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.897527694702148, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.238573431968689, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.209813117980957, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02819579653441906, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09172117710113525, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08967316150665283, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.031458672136068344, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14699597656726837, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016756756231188774, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8642085790634155, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0101079773157835, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.839092969894409, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1850.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7462934851646423, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.7011117935180664, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 22807.90625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 22807.74609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26170799136161804, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1228886.5, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07472451776266098, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7195613384246826, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.05501394718885422, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.576361656188965, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2904.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6022135615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.0509769916534424, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.024253845214844, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2356724739074707, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.11467956751585007, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.1020907461643219, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.850159168243408, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6621427536010742, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.281740188598633, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015076967887580395, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08646167814731598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07424881309270859, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.022131431847810745, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13167232275009155, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006271230522543192, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11365950852632523, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03142548352479935, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.56658935546875, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19135.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14535096287727356, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6505342125892639, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5149260759353638, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4215182363986969, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09287326782941818, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.567466735839844, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027640853077173233, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24495990574359894, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.04120412468910217, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1845533847808838, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21707.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8715951442718506, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.5156192779541016, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 7.06961727142334, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.692551612854004, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.039292410016059875, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7381612062454224, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.6208581924438477, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.39129751920700073, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.682665824890137, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01712527871131897, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08478416502475739, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.074790358543396, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.021519454196095467, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1318666636943817, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0022959010675549507, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22108671069145203, "metrics/particle/momentum/all/accepted/pz_bias": -0.026279456913471222, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.064949870109558, "metrics/particle/momentum/all/accepted/samples": 29618.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23747211694717407, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8469608426094055, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1883.8865966796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1883.7825927734375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11851009726524353, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 353121.3125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012141029350459576, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6114557385444641, "metrics/particle/momentum/all/assigned/pz_bias": -0.03184090182185173, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.2410545349121094, "metrics/particle/momentum/all/assigned/samples": 35170.0, "metrics/particle/momentum/all/efficiency": 0.8278965950012207, "metrics/particle/momentum/all/event/energy_bias": 7.228065013885498, "metrics/particle/momentum/all/event/p3_error_mean": 14.579303741455078, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.9533772468566895, "metrics/particle/momentum/all/event/px_bias": 0.6082078218460083, "metrics/particle/momentum/all/event/py_bias": -0.6849424839019775, "metrics/particle/momentum/all/event/pz_abs_mean": 11.26268482208252, "metrics/particle/momentum/all/event/pz_bias": -3.292013645172119, "metrics/particle/momentum/all/event/pz_residual_rms": 20.392240524291992, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.009368372149765491, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.039030540734529495, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020550493150949478, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0039975945837795734, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.058974288403987885, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011011194437742233, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11903740465641022, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.025220122188329697, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8024975657463074, "metrics/particle/momentum/class_1/accepted/samples": 16347.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.041068900376558304, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10295317322015762, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09028609097003937, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06365687400102615, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.018905945122241974, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6892015933990479, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0038523084949702024, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2255447655916214, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.030101655051112175, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6830066442489624, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9688259363174438, "metrics/particle/momentum/class_1/event/energy_bias": 4.561097145080566, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.945064544677734, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.001006126403809, "metrics/particle/momentum/class_1/event/px_bias": -0.6315065026283264, "metrics/particle/momentum/class_1/event/py_bias": -0.47509509325027466, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.672979354858398, "metrics/particle/momentum/class_1/event/pz_bias": -0.6202747225761414, "metrics/particle/momentum/class_1/event/pz_residual_rms": 17.982149124145508, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03919432684779167, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19530098140239716, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19559788703918457, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02171817421913147, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23802077770233154, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029202841222286224, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.599016785621643, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.010327201336622238, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.592729091644287, "metrics/particle/momentum/class_2/accepted/samples": 1267.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9328569173812866, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.720852255821228, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 24038.294921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 24037.80078125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.38039928674697876, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1261680.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07731397449970245, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.096001148223877, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0432032085955143, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.153532981872559, "metrics/particle/momentum/class_2/assigned/samples": 2755.0, "metrics/particle/momentum/class_2/efficiency": 0.43007469177246094, "metrics/particle/momentum/class_2/event/energy_bias": -4.766623497009277, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.632381439208984, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.101602554321289, "metrics/particle/momentum/class_2/event/px_bias": 0.28622135519981384, "metrics/particle/momentum/class_2/event/py_bias": -0.7608607411384583, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.062604904174805, "metrics/particle/momentum/class_2/event/pz_bias": -1.1379221677780151, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.002925872802734, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025973433628678322, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13964048027992249, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14020079374313354, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.051896996796131134, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18250839412212372, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011317140888422728, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21397477388381958, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.026861365884542465, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7506448030471802, "metrics/particle/momentum/class_3/accepted/samples": 11487.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33720123767852783, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7264437675476074, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9371269941329956, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8151196241378784, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18535804748535156, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.00714111328125, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009734631516039371, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5964916348457336, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.024400588124990463, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.4355990886688232, "metrics/particle/momentum/class_3/assigned/samples": 14998.0, "metrics/particle/momentum/class_3/efficiency": 0.7453283071517944, "metrics/particle/momentum/class_3/event/energy_bias": 8.06684684753418, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.923693656921387, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.335470676422119, "metrics/particle/momentum/class_3/event/px_bias": 0.6811832189559937, "metrics/particle/momentum/class_3/event/py_bias": 0.6061180830001831, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.232015132904053, "metrics/particle/momentum/class_3/event/pz_bias": -1.6020218133926392, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.24020767211914, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013387471437454224, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05309118330478668, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05190550163388252, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006713593378663063, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09708999842405319, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24730445444583893, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.06958409398794174, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7543111443519592, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07006338238716125, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.4564383924007416, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.5774288177490234, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.5265305042266846, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 5.6998982429504395, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3318559527397156, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.10311629623174667, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.037004828453064, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -0.2994319498538971, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.820178508758545, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1236350536346436, "metrics/particle/momentum/class_4/event/px_bias": 0.22462861239910126, "metrics/particle/momentum/class_4/event/py_bias": -0.06379769742488861, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2259273529052734, "metrics/particle/momentum/class_4/event/pz_bias": 0.32328587770462036, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.809854984283447, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008551291190087795, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020468445494771004, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018709473311901093, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006265282165259123, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05676916986703873, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19424745440483093, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.11682169139385223, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.49512946605682373, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.034124452620744705, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.033544350415468216, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03145263344049454, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00847301259636879, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11848660558462143, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5686015486717224, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.49245205521583557, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.2261152267456055, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.33382394909858704, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9152221083641052, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.575865626335144, "metrics/particle/momentum/class_5/event/px_bias": 0.04768111929297447, "metrics/particle/momentum/class_5/event/py_bias": 0.00869300588965416, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6042222380638123, "metrics/particle/momentum/class_5/event/pz_bias": -0.2550809383392334, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.498469352722168, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6339209079742432, "step_valid_time": 8235.142479896545, "step_total_time": 8236.776401281357, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0065220953413543, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06694163460353442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3029, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560911191812479, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9583057577763071, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.043908880818752064, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.953449983492902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9556585043017869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960530679933665, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0170257394687043, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08088621186744305, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2127, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9449622725255216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504464285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9476964166481193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049553571428571426, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9440745672436751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9495535714285714, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9468061428889384, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9449622725255216, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0192411305217925, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08262454710788414, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1188, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1184, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9245136186770428, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9266770670826833, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9255940786910791, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07332293291731669, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9214007782101167, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9235569422776911, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9224776003116478, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9245136186770428} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_2000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_2000.json new file mode 100644 index 0000000000000000000000000000000000000000..b1f0cbd773c292a876e7fd8bb8bea3dd27427a8b --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_2000.json @@ -0,0 +1 @@ +{"epoch": 0.09475080538184574, "train": {"Classification_binary": 3.540337324142456, "Classification": 0.03272764012217522, "Regression_pt": 0.19157558679580688, "Regression_eta": 0.01489258836954832, "Regression_sin_phi": 0.009812376461923122, "Regression_cos_phi": 0.00894648116081953, "Regression_energy": 0.19177331030368805, "Total": 12.334278106689453, "Total_unweighted": 3.990065307356417}, "train_group": {}, "valid": {"Classification_binary": 3.577398935953776, "Classification": 0.033120078345139824, "Regression_pt": 0.19113731384277344, "Regression_eta": 0.014749931792418161, "Regression_sin_phi": 0.010411397243539492, "Regression_cos_phi": 0.010452426970005035, "Regression_energy": 0.19432532787322998, "Total": 12.544147491455078, "Total_unweighted": 4.031595412020882, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03396020457148552, "diagnostic/loss/clic_pf/Classification_binary": 3.5628628730773926, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009955772198736668, "diagnostic/loss/clic_pf/Regression_energy": 0.1969701200723648, "diagnostic/loss/clic_pf/Regression_eta": 0.012965669855475426, "diagnostic/loss/clic_pf/Regression_pt": 0.1939462423324585, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.010215614922344685, "diagnostic/regression/clic_pf/energy_pred_mean": 0.07853571325540543, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2314475178718567, "diagnostic/regression/clic_pf/energy_residual_mean": 0.09408934414386749, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5229319930076599, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.04994071647524834, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.23375485837459564, "diagnostic/regression/clic_pf/pt_residual_mean": 0.11878523975610733, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5279557108879089, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9529886245727539, "metrics/particle/class_1/pid_accuracy": 0.9856910705566406, "metrics/particle/class_2/efficiency": 0.2935272753238678, "metrics/particle/class_2/pid_accuracy": 0.4341144859790802, "metrics/particle/class_3/efficiency": 0.6766677498817444, "metrics/particle/class_3/pid_accuracy": 0.9794546961784363, "metrics/particle/class_4/efficiency": 0.9331395626068115, "metrics/particle/class_4/pid_accuracy": 0.1526479721069336, "metrics/particle/class_5/efficiency": 0.9818913340568542, "metrics/particle/class_5/pid_accuracy": 0.4508196711540222, "metrics/particle/count/bias_mean": 0.5267974138259888, "metrics/particle/count/mae": 4.13725471496582, "metrics/particle/count/prediction_mean": 70.5261459350586, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.09362263232469559, "metrics/particle/event/energy_response_mean": 1.0594083070755005, "metrics/particle/event/met_abs_error": 9.537332534790039, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0859750285744667, "metrics/particle/event/scalar_pt_response_mean": 1.0622371435165405, "metrics/particle/event/vector_pt_closure": 0.07569894194602966, "metrics/particle/matched/delta_eta_abs_mean": 0.016335254535079002, "metrics/particle/matched/delta_phi_abs_mean": 0.013166910037398338, "metrics/particle/matched/delta_r_mean": 0.023347586393356323, "metrics/particle/matched/energy_relative_abs_mean": 0.09260420501232147, "metrics/particle/matched/pid_accuracy": 0.9533756971359253, "metrics/particle/matched/pt_relative_abs_mean": 0.08975760638713837, "metrics/particle/matching/duplicate_fraction": 0.04092488810420036, "metrics/particle/matching/efficiency": 0.7786253690719604, "metrics/particle/matching/f1": 0.7757064700126648, "metrics/particle/matching/purity": 0.7728093862533569, "metrics/particle/matching/target_coverage_dr0p05": 0.7215286493301392, "metrics/particle/matching/target_coverage_dr0p10": 0.8197181820869446, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6834143996238708, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8261702060699463, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.024946166202425957, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08763638883829117, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.08768459409475327, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.04229748994112015, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13723793625831604, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004282115958631039, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.36606550216674805, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.045408353209495544, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2581223249435425, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 7940.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3930083215236664, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.7171038389205933, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 2.108546495437622, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.9925639629364014, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1722644567489624, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 29.625429153442383, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.024444229900836945, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0904711484909058, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04069413244724274, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.6723663806915283, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10391.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7386733889579773, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 3.0576744079589844, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 11.752016067504883, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 5.208306312561035, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.21332590281963348, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 1.1863083839416504, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 10.138975143432617, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.927140951156616, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 18.044767379760742, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03366570919752121, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09490714967250824, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.10053078830242157, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.04665151610970497, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15348105132579803, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018613606691360474, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9598281383514404, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0944746732711792, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.040809392929077, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1558.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.894436776638031, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 3.526303291320801, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 6.412895679473877, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 6.257899284362793, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3154761791229248, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 93.79334259033203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09593837708234787, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.712017297744751, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.07821663469076157, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.692763328552246, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2856.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5071614384651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 3.1438968181610107, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 10.824368476867676, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.8223345279693604, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04714958369731903, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1839580237865448, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 10.562820434570312, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.6073768138885498, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 24.29205894470215, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.021901443600654602, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09415825456380844, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.08996764570474625, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.04120435193181038, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.14085818827152252, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0008171270019374788, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.140797421336174, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.036550167948007584, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6757124662399292, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18357.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.20061179995536804, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.2458196878433228, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9737697243690491, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8727980256080627, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.12045012414455414, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 16.3219051361084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.005557099357247353, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.39683231711387634, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0325276218354702, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 3.0032055377960205, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21594.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8361574411392212, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 9.14262580871582, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.86699104309082, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.642112731933594, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -1.2971867322921753, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.1790392398834229, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.22334623336792, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.09278102219104767, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 18.798452377319336, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.023427339270710945, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0923411101102829, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0899076983332634, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.041820622980594635, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.14057688415050507, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028002155013382435, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.25081998109817505, "metrics/particle/momentum/all/accepted/pz_bias": -0.031746625900268555, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.126621127128601, "metrics/particle/momentum/all/accepted/samples": 27855.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.31486672163009644, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.5733124017715454, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.7580641508102417, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.6481873989105225, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.15189000964164734, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 33.88209915161133, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.018598778173327446, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.875457227230072, "metrics/particle/momentum/all/assigned/pz_bias": -0.0016119519714266062, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.562837600708008, "metrics/particle/momentum/all/assigned/samples": 34841.0, "metrics/particle/momentum/all/efficiency": 0.7786163687705994, "metrics/particle/momentum/all/event/energy_bias": 15.344198226928711, "metrics/particle/momentum/all/event/p3_error_mean": 25.194303512573242, "metrics/particle/momentum/all/event/pt_vector_error_mean": 13.157750129699707, "metrics/particle/momentum/all/event/px_bias": -1.0367112159729004, "metrics/particle/momentum/all/event/py_bias": 2.5493059158325195, "metrics/particle/momentum/all/event/pz_abs_mean": 18.430837631225586, "metrics/particle/momentum/all/event/pz_bias": -1.412545084953308, "metrics/particle/momentum/all/event/pz_residual_rms": 31.841609954833984, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014663327485322952, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04590928927063942, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.042176470160484314, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.029981836676597595, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.07282156497240067, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014307931996881962, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.15699028968811035, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.03589495271444321, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8434586524963379, "metrics/particle/momentum/class_1/accepted/samples": 16075.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0650697872042656, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.18349589407444, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.18020732700824738, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15459197759628296, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03069993481040001, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.062445878982544, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005452497862279415, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3428823947906494, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.0263795405626297, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.600680112838745, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9527055025100708, "metrics/particle/momentum/class_1/event/energy_bias": 9.852216720581055, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.656207084655762, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 7.934901237487793, "metrics/particle/momentum/class_1/event/px_bias": -1.5859291553497314, "metrics/particle/momentum/class_1/event/py_bias": -0.6860238313674927, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.697552680969238, "metrics/particle/momentum/class_1/event/pz_bias": -1.7003505229949951, "metrics/particle/momentum/class_1/event/pz_residual_rms": 25.08858871459961, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04645548388361931, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21646590530872345, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2150806486606598, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05646670237183571, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2546212673187256, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0405244342982769, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8232343196868896, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04202185198664665, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.9991824626922607, "metrics/particle/momentum/class_2/accepted/samples": 839.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.1967815160751343, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.1105157136917114, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.335181474685669, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.7432673573493958, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.49885669350624084, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.087444305419922, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.11051829159259796, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.229511737823486, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.010935882106423378, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.312968254089355, "metrics/particle/momentum/class_2/assigned/samples": 2624.0, "metrics/particle/momentum/class_2/efficiency": 0.2847929298877716, "metrics/particle/momentum/class_2/event/energy_bias": -8.768354415893555, "metrics/particle/momentum/class_2/event/p3_error_mean": 22.92597007751465, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 11.530327796936035, "metrics/particle/momentum/class_2/event/px_bias": 0.5750564336776733, "metrics/particle/momentum/class_2/event/py_bias": 1.771545171737671, "metrics/particle/momentum/class_2/event/pz_abs_mean": 17.19518280029297, "metrics/particle/momentum/class_2/event/pz_bias": -0.696740984916687, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.760889053344727, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03546777367591858, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.15622439980506897, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.15545429289340973, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.06911229342222214, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1977183222770691, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.002015161793678999, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.26542943716049194, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.028932612389326096, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.9777518510818481, "metrics/particle/momentum/class_3/accepted/samples": 10421.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.45303472876548767, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 3.2781434059143066, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 3.681941032409668, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 3.558778762817383, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.23283784091472626, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 51.892181396484375, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.017905404791235924, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9037649631500244, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.034985680133104324, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.321899890899658, "metrics/particle/momentum/class_3/assigned/samples": 14800.0, "metrics/particle/momentum/class_3/efficiency": 0.6761614084243774, "metrics/particle/momentum/class_3/event/energy_bias": 16.170406341552734, "metrics/particle/momentum/class_3/event/p3_error_mean": 15.629924774169922, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 8.015640258789062, "metrics/particle/momentum/class_3/event/px_bias": -0.6027431488037109, "metrics/particle/momentum/class_3/event/py_bias": 1.2594223022460938, "metrics/particle/momentum/class_3/event/pz_abs_mean": 11.650064468383789, "metrics/particle/momentum/class_3/event/pz_bias": 0.42264872789382935, "metrics/particle/momentum/class_3/event/pz_residual_rms": 20.538185119628906, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01753816567361355, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05335598066449165, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05568108335137367, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.015004764311015606, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09010359644889832, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.32721996307373047, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.05935760587453842, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.0822944641113281, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06795397400856018, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.773735761642456, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.5695555210113525, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.526880145072937, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.423515558242798, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3767005503177643, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0794219896197319, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2088979482650757, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -1.6006295680999756, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.4940552711486816, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.6140528917312622, "metrics/particle/momentum/class_4/event/px_bias": 0.03709326311945915, "metrics/particle/momentum/class_4/event/py_bias": 0.10507892817258835, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.610610842704773, "metrics/particle/momentum/class_4/event/pz_bias": 0.38685619831085205, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.919314384460449, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01276206411421299, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.035351645201444626, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.03880399465560913, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.02797805517911911, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06123034656047821, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3108355402946472, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.11651787161827087, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8803685307502747, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.013635203242301941, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.04968266934156418, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05304704234004021, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02787146158516407, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11956321448087692, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5825473666191101, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.39231666922569275, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.36137318611145, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.309440940618515, "metrics/particle/momentum/class_5/event/p3_error_mean": 2.4428420066833496, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.7449179887771606, "metrics/particle/momentum/class_5/event/px_bias": 0.5398115515708923, "metrics/particle/momentum/class_5/event/py_bias": 0.0992831140756607, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.351680874824524, "metrics/particle/momentum/class_5/event/pz_bias": 0.17504140734672546, "metrics/particle/momentum/class_5/event/pz_residual_rms": 5.699641704559326, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.3432676792144775, "step_valid_time": 873.9778153896332, "step_total_time": 875.3210835456848, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0408704592154518, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1411040698540601, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3037, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2813, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2763, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9330016583747927, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9262430029634507, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9296100462656973, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07375699703654923, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9164179104477612, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9097793875535067, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9130865829477859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9330016583747927, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0354726371690384, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15018308549144888, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2023, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9081225033288948, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.910547396528705, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9093333333333333, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08945260347129506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8979138925876609, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9003115264797508, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8991111111111111, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9081225033288948, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0461918146847067, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16888336037852858, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1289, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1132, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1099, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8809338521400778, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.878200155159038, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8795648795648796, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12179984484096198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8552529182879377, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8525989138867339, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.853923853923854, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8809338521400778} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_20000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..c3ddfc3816aeb614dea6a13e1b55937378baff84 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.9475080538184575, "train": {"Classification_binary": 2.630510091781616, "Classification": 0.021744193509221077, "Regression_pt": 0.12620887160301208, "Regression_eta": 0.009793194942176342, "Regression_sin_phi": 0.007732273545116186, "Regression_cos_phi": 0.007901844568550587, "Regression_energy": 0.1302773654460907, "Total": 8.691519737243652, "Total_unweighted": 2.934167835395783}, "train_group": {}, "valid": {"Classification_binary": 2.529696464538574, "Classification": 0.020485098163286846, "Regression_pt": 0.13048591216405234, "Regression_eta": 0.010451233635346094, "Regression_sin_phi": 0.008246553440888723, "Regression_cos_phi": 0.008232104902466139, "Regression_energy": 0.13389445344607034, "Total": 8.736038208007812, "Total_unweighted": 2.8414918202906847, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02100290358066559, "diagnostic/loss/clic_pf/Classification_binary": 2.5399093627929688, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007782408036291599, "diagnostic/loss/clic_pf/Regression_energy": 0.13831676542758942, "diagnostic/loss/clic_pf/Regression_eta": 0.009791887365281582, "diagnostic/loss/clic_pf/Regression_pt": 0.13518457114696503, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007801800966262817, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03886207565665245, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19181592762470245, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05441569909453392, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43321865797042847, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.004485609475523233, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18465690314769745, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07333015650510788, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43410709500312805, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9682350158691406, "metrics/particle/class_1/pid_accuracy": 0.9877640604972839, "metrics/particle/class_2/efficiency": 0.43797925114631653, "metrics/particle/class_2/pid_accuracy": 0.7435633540153503, "metrics/particle/class_3/efficiency": 0.751481831073761, "metrics/particle/class_3/pid_accuracy": 0.9857450127601624, "metrics/particle/class_4/efficiency": 0.9244186282157898, "metrics/particle/class_4/pid_accuracy": 0.472746342420578, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6639510989189148, "metrics/particle/count/bias_mean": -0.5660130977630615, "metrics/particle/count/mae": 3.1516339778900146, "metrics/particle/count/prediction_mean": 69.43333435058594, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04925309121608734, "metrics/particle/event/energy_response_mean": 1.0101048946380615, "metrics/particle/event/met_abs_error": 4.401476860046387, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0383889339864254, "metrics/particle/event/scalar_pt_response_mean": 1.0115197896957397, "metrics/particle/event/vector_pt_closure": 0.04013657569885254, "metrics/particle/matched/delta_eta_abs_mean": 0.009273907169699669, "metrics/particle/matched/delta_phi_abs_mean": 0.010014977306127548, "metrics/particle/matched/delta_r_mean": 0.015331199392676353, "metrics/particle/matched/energy_relative_abs_mean": 0.08490118384361267, "metrics/particle/matched/pid_accuracy": 0.9689886569976807, "metrics/particle/matched/pt_relative_abs_mean": 0.07499027252197266, "metrics/particle/matching/duplicate_fraction": 0.027882108464837074, "metrics/particle/matching/efficiency": 0.8301011323928833, "metrics/particle/matching/f1": 0.8334708213806152, "metrics/particle/matching/purity": 0.8368679881095886, "metrics/particle/matching/target_coverage_dr0p05": 0.8065808415412903, "metrics/particle/matching/target_coverage_dr0p10": 0.8630239367485046, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7358518838882446, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8663105964660645, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01805187575519085, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08178363740444183, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07481026649475098, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018679501488804817, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13152723014354706, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030005769804120064, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3197367787361145, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.010939056053757668, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.164069414138794, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8665.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27338266372680664, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8170136213302612, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8294498324394226, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7209662795066833, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12102889269590378, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.631911277770996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011997312307357788, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7122322916984558, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.028879739344120026, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.046121120452881, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10419.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8061215281486511, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0871984958648682, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.1756205558776855, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6050307750701904, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.25105687975883484, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.0258494783192873, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.397003650665283, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.6309286952018738, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.61673355102539, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026756079867482185, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08714349567890167, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08472061902284622, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01930195279419422, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14045387506484985, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01950162462890148, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8322300314903259, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.013639840297400951, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6916167736053467, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1846.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7359042763710022, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.982431173324585, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.8184165954589844, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6659865379333496, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2543921172618866, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.76923942565918, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0783555880188942, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.9309349060058594, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.027271179482340813, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.790984153747559, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2846.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6009114384651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4669136703014374, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.8727569580078125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1122901439666748, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12306901067495346, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.050795938819646835, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.717883586883545, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7345353364944458, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.803483009338379, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013229734264314175, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08669040352106094, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07506532967090607, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01967286318540573, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13218651711940765, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004689698398578912, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10798166692256927, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.010775006376206875, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5547301173210144, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19191.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14084899425506592, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7051457166671753, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5153660178184509, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.42018723487854004, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08518724143505096, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.824609279632568, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025027808733284473, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24230752885341644, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.017077721655368805, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.510489821434021, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21576.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8741459250450134, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.45854097604751587, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.5884904861450195, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.621312141418457, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.04104766249656677, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5092232823371887, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.740391731262207, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.15847653150558472, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.1222710609436035, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015477175824344158, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08528710156679153, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07559100538492203, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01936001516878605, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13252413272857666, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002390411449596286, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21476976573467255, "metrics/particle/momentum/all/accepted/pz_bias": 0.011000916361808777, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.021963357925415, "metrics/particle/momentum/all/accepted/samples": 29702.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22908975183963776, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8429346680641174, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7974165081977844, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6935824751853943, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10972704738378525, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.073877334594727, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011538130231201649, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6024573445320129, "metrics/particle/momentum/all/assigned/pz_bias": 0.016984393820166588, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.466747283935547, "metrics/particle/momentum/all/assigned/samples": 34841.0, "metrics/particle/momentum/all/efficiency": 0.8302446007728577, "metrics/particle/momentum/all/event/energy_bias": 1.078825831413269, "metrics/particle/momentum/all/event/p3_error_mean": 13.308821678161621, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.975948333740234, "metrics/particle/momentum/all/event/px_bias": 0.16903550922870636, "metrics/particle/momentum/all/event/py_bias": -0.43257787823677063, "metrics/particle/momentum/all/event/pz_abs_mean": 10.497210502624512, "metrics/particle/momentum/all/event/pz_bias": 0.054869890213012695, "metrics/particle/momentum/all/event/pz_residual_rms": 18.02056121826172, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008472017012536526, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0387081615626812, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0210036039352417, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005159944295883179, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.059841737151145935, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011034144554287195, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10871073603630066, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.025996888056397438, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7070426940917969, "metrics/particle/momentum/class_1/accepted/samples": 16313.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04480616748332977, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13447342813014984, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12862402200698853, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10178529471158981, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02020980231463909, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.2662923336029053, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004444971214979887, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23540306091308594, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.030573226511478424, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9804904460906982, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9668108820915222, "metrics/particle/momentum/class_1/event/energy_bias": 3.3073909282684326, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.509115219116211, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.695537567138672, "metrics/particle/momentum/class_1/event/px_bias": -0.45490944385528564, "metrics/particle/momentum/class_1/event/py_bias": -0.39179933071136475, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.31685209274292, "metrics/particle/momentum/class_1/event/pz_bias": 0.4249889552593231, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.841849327087402, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038709986954927444, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19593288004398346, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1985609531402588, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.028620276600122452, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24042271077632904, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030651340261101723, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5675572156906128, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.005217494443058968, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3938498497009277, "metrics/particle/momentum/class_2/accepted/samples": 1305.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9038078784942627, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6578911542892456, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8492481112480164, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3692076802253723, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34992459416389465, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.112542629241943, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06900452822446823, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9994492530822754, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.11154278367757797, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.717313289642334, "metrics/particle/momentum/class_2/assigned/samples": 2652.0, "metrics/particle/momentum/class_2/efficiency": 0.4429735243320465, "metrics/particle/momentum/class_2/event/energy_bias": -6.949295520782471, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.788039207458496, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.792048454284668, "metrics/particle/momentum/class_2/event/px_bias": 0.1313343048095703, "metrics/particle/momentum/class_2/event/py_bias": -0.5130236744880676, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.500751495361328, "metrics/particle/momentum/class_2/event/pz_bias": -0.14724476635456085, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.636211395263672, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0229740459471941, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14043711125850677, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14022941887378693, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04542165249586105, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18221139907836914, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011232071556150913, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21117383241653442, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008782013319432735, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.8051656484603882, "metrics/particle/momentum/class_3/accepted/samples": 11574.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.32427698373794556, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.664506435394287, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.532564401626587, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4064536094665527, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1709991842508316, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.096643447875977, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.00961278099566698, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5982059836387634, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.012980223633348942, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.5944907665252686, "metrics/particle/momentum/class_3/assigned/samples": 14772.0, "metrics/particle/momentum/class_3/efficiency": 0.7509732842445374, "metrics/particle/momentum/class_3/event/energy_bias": 5.413539886474609, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.38331413269043, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.001653671264648, "metrics/particle/momentum/class_3/event/px_bias": 0.16362783312797546, "metrics/particle/momentum/class_3/event/py_bias": 0.42497718334198, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.943816184997559, "metrics/particle/momentum/class_3/event/pz_bias": -0.39062613248825073, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.572182655334473, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011778987012803555, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05213753506541252, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0522858090698719, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004675737116485834, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09664089977741241, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2567153871059418, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.029110591858625412, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8550463914871216, "metrics/particle/momentum/class_4/accepted/samples": 332.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08314826339483261, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1000165939331055, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.9750287532806396, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.911603331565857, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.06629834324121475, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 32.206077575683594, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.34741631150245667, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.027353579178452492, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0859125852584839, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9171270728111267, "metrics/particle/momentum/class_4/event/energy_bias": -0.14348916709423065, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.876096248626709, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1497399806976318, "metrics/particle/momentum/class_4/event/px_bias": 0.17520147562026978, "metrics/particle/momentum/class_4/event/py_bias": -0.023751873522996902, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2661645412445068, "metrics/particle/momentum/class_4/event/pz_bias": 0.36367902159690857, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.865896224975586, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0065746731124818325, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01870940811932087, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017280597239732742, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005302892532199621, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04893534258008003, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17233698070049286, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.008141129277646542, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7058690190315247, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04660898819565773, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.036792926490306854, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.034659795463085175, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.002259559463709593, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1318982094526291, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5563575625419617, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2092222273349762, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.26960563659668, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.5493204593658447, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9179503321647644, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6128698587417603, "metrics/particle/momentum/class_5/event/px_bias": 0.15378135442733765, "metrics/particle/momentum/class_5/event/py_bias": 0.07101987302303314, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5735019445419312, "metrics/particle/momentum/class_5/event/pz_bias": -0.1959272027015686, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.330470323562622, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7486937046051025, "step_valid_time": 8670.090727329254, "step_total_time": 8671.839421510696, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0046020785452867, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06364681434402697, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954800395908941, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.957327158451869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045199604091059054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9575456053067993, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9524909270867701, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9550115779027456, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9598673300165838, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067215053369485, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07504217460772267, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2125, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2120, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9431868619618287, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.947814451382694, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9454949944382648, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052185548617305975, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409675987572126, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9455842997323818, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9432703003337041, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9431868619618287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006987036579511, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07311062959545944, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1174, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1171, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9136186770428015, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.918622848200313, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9161139289894654, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.081377151799687, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.911284046692607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9162754303599374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9137729223566133, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9136186770428015} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_21000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_21000.json new file mode 100644 index 0000000000000000000000000000000000000000..4787c72a475e13142d7ce46074208493851c357d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_21000.json @@ -0,0 +1 @@ +{"epoch": 0.9948834565093804, "train": {"Classification_binary": 2.6127309799194336, "Classification": 0.020063934847712517, "Regression_pt": 0.13148877024650574, "Regression_eta": 0.01011836901307106, "Regression_sin_phi": 0.009389335289597511, "Regression_cos_phi": 0.007891816087067127, "Regression_energy": 0.12986665964126587, "Total": 8.786968231201172, "Total_unweighted": 2.9215498650446534}, "train_group": {}, "valid": {"Classification_binary": 2.5456391970316568, "Classification": 0.02036542942126592, "Regression_pt": 0.13103530804316202, "Regression_eta": 0.010650120675563812, "Regression_sin_phi": 0.008263147746523222, "Regression_cos_phi": 0.00826035812497139, "Regression_energy": 0.1345017353693644, "Total": 8.776235580444336, "Total_unweighted": 2.8587152964125075, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020965468138456345, "diagnostic/loss/clic_pf/Classification_binary": 2.5630452632904053, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007816512137651443, "diagnostic/loss/clic_pf/Regression_energy": 0.13789549469947815, "diagnostic/loss/clic_pf/Regression_eta": 0.009953969158232212, "diagnostic/loss/clic_pf/Regression_pt": 0.1348203718662262, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007877481169998646, "diagnostic/regression/clic_pf/energy_pred_mean": 0.029784373939037323, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1903102993965149, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04533799737691879, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4315331280231476, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0013036185409873724, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1847333312034607, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06754092127084732, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43414223194122314, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9686723351478577, "metrics/particle/class_1/pid_accuracy": 0.9869692921638489, "metrics/particle/class_2/efficiency": 0.4261389374732971, "metrics/particle/class_2/pid_accuracy": 0.750198483467102, "metrics/particle/class_3/efficiency": 0.7475805878639221, "metrics/particle/class_3/pid_accuracy": 0.983738899230957, "metrics/particle/class_4/efficiency": 0.932170569896698, "metrics/particle/class_4/pid_accuracy": 0.48232847452163696, "metrics/particle/class_5/efficiency": 0.9959758520126343, "metrics/particle/class_5/pid_accuracy": 0.7272727489471436, "metrics/particle/count/bias_mean": -0.5209150314331055, "metrics/particle/count/mae": 3.1797385215759277, "metrics/particle/count/prediction_mean": 69.47843170166016, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04868684709072113, "metrics/particle/event/energy_response_mean": 1.002482533454895, "metrics/particle/event/met_abs_error": 4.437723159790039, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036853667348623276, "metrics/particle/event/scalar_pt_response_mean": 1.004315733909607, "metrics/particle/event/vector_pt_closure": 0.0404491163790226, "metrics/particle/matched/delta_eta_abs_mean": 0.010121080093085766, "metrics/particle/matched/delta_phi_abs_mean": 0.01178352814167738, "metrics/particle/matched/delta_r_mean": 0.01727616973221302, "metrics/particle/matched/energy_relative_abs_mean": 0.08266923576593399, "metrics/particle/matched/pid_accuracy": 0.9686862230300903, "metrics/particle/matched/pt_relative_abs_mean": 0.07364977151155472, "metrics/particle/matching/duplicate_fraction": 0.02691388688981533, "metrics/particle/matching/efficiency": 0.82774817943573, "metrics/particle/matching/f1": 0.8308395743370056, "metrics/particle/matching/purity": 0.8339542150497437, "metrics/particle/matching/target_coverage_dr0p05": 0.7965153455734253, "metrics/particle/matching/target_coverage_dr0p10": 0.8591023087501526, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.739512026309967, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8667401075363159, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018911560997366905, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07772241532802582, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07209105044603348, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014851813204586506, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12728162109851837, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003612632630392909, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31540149450302124, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.033823732286691666, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1942170858383179, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8581.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28254234790802, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8277125358581543, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8892769813537598, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7739852070808411, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12392013520002365, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.841556549072266, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013630255125463009, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7103331685066223, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01159270666539669, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.653062343597412, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10418.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7983068227767944, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5711177587509155, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.065098285675049, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6972765922546387, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.08378687500953674, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1797616332769394, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.229035377502441, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.1871267706155777, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.032931327819824, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02772090770304203, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08799510449171066, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08847108483314514, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02224889025092125, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14421533048152924, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018121911212801933, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8774425983428955, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.009891954250633717, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.024730920791626, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1821.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.721581220626831, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.505617618560791, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9875.6474609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9875.4873046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2494729459285736, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 526703.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07273366302251816, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.7025699615478516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08218862116336823, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.733224868774414, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2846.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5927734375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.915054202079773, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.255253791809082, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1208281517028809, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.007035344839096069, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.061242613941431046, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.08198881149292, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.6138094663619995, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.707592010498047, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.015725957229733467, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08459072560071945, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07333804666996002, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01284689549356699, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1293925642967224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005739330081269145, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10950687527656555, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.012750629335641861, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5729341506958008, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19166.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14732377231121063, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5653094053268433, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.438264399766922, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3336522877216339, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08439719676971436, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.622923851013184, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027853860519826412, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23574471473693848, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.007427430246025324, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2906088829040527, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21541.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8730071783065796, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.6705973148345947, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.856578826904297, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.850844860076904, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.11051388084888458, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.09823863208293915, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.786534070968628, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.21542605757713318, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.304039001464844, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.017389187589287758, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08280711621046066, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07390815019607544, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014007787220180035, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1297496259212494, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025365259498357773, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21655480563640594, "metrics/particle/momentum/all/accepted/pz_bias": -0.01747182011604309, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0909260511398315, "metrics/particle/momentum/all/accepted/samples": 29568.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23475497961044312, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7207419872283936, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 808.0676879882812, "metrics/particle/momentum/all/assigned/pt_relative_bias": 807.9554443359375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10972561687231064, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 150613.234375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011751185171306133, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5795127749443054, "metrics/particle/momentum/all/assigned/pz_bias": -0.007847445085644722, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8333418369293213, "metrics/particle/momentum/all/assigned/samples": 34805.0, "metrics/particle/momentum/all/efficiency": 0.8264989256858826, "metrics/particle/momentum/all/event/energy_bias": -2.156769275665283, "metrics/particle/momentum/all/event/p3_error_mean": 12.118572235107422, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.006802558898926, "metrics/particle/momentum/all/event/px_bias": 0.2013361155986786, "metrics/particle/momentum/all/event/py_bias": 0.3392428755760193, "metrics/particle/momentum/all/event/pz_abs_mean": 9.086912155151367, "metrics/particle/momentum/all/event/pz_bias": -1.58551025390625, "metrics/particle/momentum/all/event/pz_residual_rms": 14.82034683227539, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008754595182836056, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.038153234869241714, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021550344303250313, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007381577510386705, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.060499731451272964, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012856618268415332, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10891079157590866, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.021997004747390747, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7662672400474548, "metrics/particle/momentum/class_1/accepted/samples": 16334.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0438920259475708, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11584675312042236, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12145331501960754, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.09408927708864212, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02015053667128086, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.468124866485596, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004859835375100374, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.23042844235897064, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.02772955782711506, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.8791265487670898, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9680554866790771, "metrics/particle/momentum/class_1/event/energy_bias": 2.580712080001831, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.0541410446167, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.62666654586792, "metrics/particle/momentum/class_1/event/px_bias": -0.594316840171814, "metrics/particle/momentum/class_1/event/py_bias": -0.07893103361129761, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.9078569412231445, "metrics/particle/momentum/class_1/event/pz_bias": -0.7635251879692078, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.968889236450195, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03926695138216019, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19756552577018738, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19795666635036469, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.04827754572033882, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23869618773460388, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03341288864612579, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7037867307662964, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05458104610443115, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.892575740814209, "metrics/particle/momentum/class_2/accepted/samples": 1257.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9278963804244995, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7296634912490845, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 10705.044921875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 10704.51171875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36533331871032715, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 548427.625, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07504761964082718, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.1041948795318604, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10428877174854279, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.242036819458008, "metrics/particle/momentum/class_2/assigned/samples": 2625.0, "metrics/particle/momentum/class_2/efficiency": 0.42668023705482483, "metrics/particle/momentum/class_2/event/energy_bias": -7.706886291503906, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.1110258102417, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.792142391204834, "metrics/particle/momentum/class_2/event/px_bias": 0.14981935918331146, "metrics/particle/momentum/class_2/event/py_bias": -0.02224346250295639, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.669910430908203, "metrics/particle/momentum/class_2/event/pz_bias": -0.5946829319000244, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.066734313964844, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027571631595492363, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1356828659772873, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1363554298877716, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.030267242342233658, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17788219451904297, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001046937657520175, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20586809515953064, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004872357472777367, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7369988560676575, "metrics/particle/momentum/class_3/accepted/samples": 11462.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.33570554852485657, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4267467260360718, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4832847118377686, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3461533784866333, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1693422794342041, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.118250846862793, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008670324459671974, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5353890061378479, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0009207456605508924, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.305126667022705, "metrics/particle/momentum/class_3/assigned/samples": 14763.0, "metrics/particle/momentum/class_3/efficiency": 0.743706226348877, "metrics/particle/momentum/class_3/event/energy_bias": 3.516746997833252, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.570123672485352, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.660898208618164, "metrics/particle/momentum/class_3/event/px_bias": 0.40644583106040955, "metrics/particle/momentum/class_3/event/py_bias": 0.36581119894981384, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.19220495223999, "metrics/particle/momentum/class_3/event/pz_bias": -0.0668952614068985, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.034197807312012, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012583237141370773, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.052497509866952896, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05240951478481293, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.014655488543212414, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.096014603972435, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2736023962497711, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.034962449222803116, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8978447914123535, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08861847221851349, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.4049200713634491, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.2769432067871094, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.22581785917282104, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 1.7393101453781128, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3699938952922821, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.019620126113295555, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2229572534561157, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.16153398156166077, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8784058094024658, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1841803789138794, "metrics/particle/momentum/class_4/event/px_bias": 0.1710539013147354, "metrics/particle/momentum/class_4/event/py_bias": 0.03416074439883232, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.2467761039733887, "metrics/particle/momentum/class_4/event/pz_bias": 0.14333394169807434, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.553586006164551, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.008700051344931126, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02290281280875206, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02232329547405243, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.013687290251255035, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05782218277454376, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.17314834892749786, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.11744417250156403, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5608389973640442, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03421943262219429, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03151678666472435, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0308496356010437, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.011579899117350578, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10323218256235123, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5248610973358154, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.46976906061172485, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1721954345703125, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.38580867648124695, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8548597097396851, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5688384771347046, "metrics/particle/momentum/class_5/event/px_bias": 0.06833383440971375, "metrics/particle/momentum/class_5/event/py_bias": 0.040445439517498016, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.538846492767334, "metrics/particle/momentum/class_5/event/pz_bias": -0.3037407100200653, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3229973316192627, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6301920413970947, "step_valid_time": 9107.158782958984, "step_total_time": 9108.7889752388, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000838377914641, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0638554141905372, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2883, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9543952412425645, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561330905479225, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04560475875743556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9562189054726368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9527428949107734, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9544777354742592, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9578772802653399, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00151890488117, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07738433098828723, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2234, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2110, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9383044829116733, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.946284691136974, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9422776911076443, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05371530886302596, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9365290723479804, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9444941808415398, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9404947626476488, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9383044829116733, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998857683207938, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07064479498887377, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1170, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.91284046692607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9121306376360808, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.912485414235706, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08786936236391912, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9105058365758755, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9097978227060654, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9101516919486581, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.91284046692607} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_22000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_22000.json new file mode 100644 index 0000000000000000000000000000000000000000..f22bfa5a130e5efac00216cd7415c7986c0fd30f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_22000.json @@ -0,0 +1 @@ +{"epoch": 1.0422588592003033, "train": {"Classification_binary": 2.511291027069092, "Classification": 0.02162943407893181, "Regression_pt": 0.14966939389705658, "Regression_eta": 0.00992133840918541, "Regression_sin_phi": 0.008610451593995094, "Regression_cos_phi": 0.007848463952541351, "Regression_energy": 0.1525070071220398, "Total": 9.280694007873535, "Total_unweighted": 2.861477116122842}, "train_group": {}, "valid": {"Classification_binary": 2.5219993591308594, "Classification": 0.020337163160244625, "Regression_pt": 0.1295273502667745, "Regression_eta": 0.010467257350683212, "Regression_sin_phi": 0.008221948519349098, "Regression_cos_phi": 0.008276171361406645, "Regression_energy": 0.13270511229832968, "Total": 8.693599065144857, "Total_unweighted": 2.831534362087647, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02080259844660759, "diagnostic/loss/clic_pf/Classification_binary": 2.528881549835205, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007903256453573704, "diagnostic/loss/clic_pf/Regression_energy": 0.13602086901664734, "diagnostic/loss/clic_pf/Regression_eta": 0.009932272136211395, "diagnostic/loss/clic_pf/Regression_pt": 0.13282731175422668, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007848167791962624, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04016494378447533, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1916867047548294, "diagnostic/regression/clic_pf/energy_residual_mean": 0.055718567222356796, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43584975600242615, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.006049222778528929, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18416428565979004, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0748937577009201, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4363715350627899, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9672212600708008, "metrics/particle/class_1/pid_accuracy": 0.9871758222579956, "metrics/particle/class_2/efficiency": 0.42568787932395935, "metrics/particle/class_2/pid_accuracy": 0.7499337792396545, "metrics/particle/class_3/efficiency": 0.7444552183151245, "metrics/particle/class_3/pid_accuracy": 0.9852051138877869, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.45188283920288086, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7444218993186951, "metrics/particle/count/bias_mean": -1.5111111402511597, "metrics/particle/count/mae": 3.4104576110839844, "metrics/particle/count/prediction_mean": 68.48823547363281, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04814630001783371, "metrics/particle/event/energy_response_mean": 1.0043389797210693, "metrics/particle/event/met_abs_error": 4.721278190612793, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037685465067625046, "metrics/particle/event/scalar_pt_response_mean": 1.0073575973510742, "metrics/particle/event/vector_pt_closure": 0.041462190449237823, "metrics/particle/matched/delta_eta_abs_mean": 0.0102623850107193, "metrics/particle/matched/delta_phi_abs_mean": 0.010352983139455318, "metrics/particle/matched/delta_r_mean": 0.016407696530222893, "metrics/particle/matched/energy_relative_abs_mean": 0.08363926410675049, "metrics/particle/matched/pid_accuracy": 0.9691362977027893, "metrics/particle/matched/pt_relative_abs_mean": 0.07325015217065811, "metrics/particle/matching/duplicate_fraction": 0.025966960936784744, "metrics/particle/matching/efficiency": 0.8256006240844727, "metrics/particle/matching/f1": 0.8346091508865356, "metrics/particle/matching/purity": 0.8438165187835693, "metrics/particle/matching/target_coverage_dr0p05": 0.7974210977554321, "metrics/particle/matching/target_coverage_dr0p10": 0.8575617074966431, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7362627387046814, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.861091136932373, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018744416534900665, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07939299941062927, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07200125604867935, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019349802285432816, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12763309478759766, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00245069433003664, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2994508147239685, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0025973974261432886, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1063610315322876, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8569.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.270737886428833, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.102211833000183, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8611505627632141, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7544105648994446, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11903835088014603, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.717772960662842, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01401596236974001, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7357370257377625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01202960405498743, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.3364064693450928, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10274.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7971904277801514, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.7082469463348389, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.088469505310059, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.74005126953125, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.1476820856332779, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0598740316927433, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.237952709197998, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5796953439712524, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.902749061584473, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.029507188126444817, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09072750061750412, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08574439585208893, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021550701931118965, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14090009033679962, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017553484067320824, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8641263842582703, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0313570499420166, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.839515209197998, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1823.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7139918208122253, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.6356775760650635, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.895374059677124, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.753124475479126, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23908375203609467, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 30.364303588867188, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07695060968399048, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.8594772815704346, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.3245198130607605, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.91994857788086, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2794.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5934244990348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.32440757751464844, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.594367027282715, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1172312498092651, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.15062689781188965, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.05756629630923271, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.448678016662598, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6359663009643555, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.542911529541016, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01422113087028265, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08477825671434402, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07265935838222504, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01749805361032486, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12900015711784363, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000732830842025578, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10951393842697144, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.022937651723623276, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5887702107429504, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19104.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13707901537418365, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6154095530509949, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.46404775977134705, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.37247800827026367, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0852481946349144, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.576500415802002, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003076205961406231, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.24356777966022491, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.008233624510467052, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5294418334960938, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21455.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8701831102371216, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.288576364517212, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.417575836181641, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.244784355163574, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1157970279455185, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.7572014331817627, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.220255136489868, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.23498055338859558, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.844274997711182, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.016479963436722755, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08358145505189896, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07327688485383987, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.018286487087607384, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1293739527463913, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00227149436250329, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21133208274841309, "metrics/particle/momentum/all/accepted/pz_bias": -0.013672839850187302, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0384806394577026, "metrics/particle/momentum/all/accepted/samples": 29496.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22354617714881897, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8428531885147095, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7789959907531738, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6788100004196167, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10775425285100937, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.567421913146973, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012310633435845375, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6017463803291321, "metrics/particle/momentum/all/assigned/pz_bias": 0.02472693845629692, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.3475136756896973, "metrics/particle/momentum/all/assigned/samples": 34523.0, "metrics/particle/momentum/all/efficiency": 0.8244863748550415, "metrics/particle/momentum/all/event/energy_bias": -0.7440782189369202, "metrics/particle/momentum/all/event/p3_error_mean": 13.482975006103516, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.668468952178955, "metrics/particle/momentum/all/event/px_bias": -0.18251192569732666, "metrics/particle/momentum/all/event/py_bias": -0.7595092058181763, "metrics/particle/momentum/all/event/pz_abs_mean": 10.227936744689941, "metrics/particle/momentum/all/event/pz_bias": 0.9806810617446899, "metrics/particle/momentum/all/event/pz_residual_rms": 17.628389358520508, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008416946046054363, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03927738964557648, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021045362576842308, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004862050525844097, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.061175793409347534, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001166789443232119, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11379101872444153, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.012255256995558739, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8032951354980469, "metrics/particle/momentum/class_1/accepted/samples": 16284.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04471465200185776, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12083790451288223, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09806916862726212, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07012677937746048, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022522522136569023, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.5840505361557007, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004208155442029238, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2271534949541092, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0023125512525439262, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.635089635848999, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9650921821594238, "metrics/particle/momentum/class_1/event/energy_bias": 2.527204751968384, "metrics/particle/momentum/class_1/event/p3_error_mean": 9.126852035522461, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.587281227111816, "metrics/particle/momentum/class_1/event/px_bias": -0.46986186504364014, "metrics/particle/momentum/class_1/event/py_bias": -0.2331785261631012, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.985341548919678, "metrics/particle/momentum/class_1/event/pz_bias": -0.13671302795410156, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.985247611999512, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038541778922080994, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19153794646263123, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19399605691432953, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019243309274315834, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2358284443616867, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029695024713873863, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5401356220245361, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.00696502486243844, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5540597438812256, "metrics/particle/momentum/class_2/accepted/samples": 1246.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9036670923233032, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.784186065196991, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9353145956993103, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.46316391229629517, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36289381980895996, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.797881126403809, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08090237528085709, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.2055866718292236, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.2765055000782013, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.035478591918945, "metrics/particle/momentum/class_2/assigned/samples": 2571.0, "metrics/particle/momentum/class_2/efficiency": 0.42294636368751526, "metrics/particle/momentum/class_2/event/energy_bias": -7.947678565979004, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.919179916381836, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.991781234741211, "metrics/particle/momentum/class_2/event/px_bias": 0.09228658676147461, "metrics/particle/momentum/class_2/event/py_bias": -0.6255943775177002, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.451885223388672, "metrics/particle/momentum/class_2/event/pz_bias": 1.4046415090560913, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.913089752197266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025795860216021538, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13655860722064972, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13565975427627563, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0417550727725029, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17697423696517944, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009606147650629282, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2037542313337326, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.013006708584725857, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6733143329620361, "metrics/particle/momentum/class_3/accepted/samples": 11451.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31611397862434387, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6782217025756836, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5305746793746948, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4100438356399536, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16407540440559387, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.685839653015137, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009906439110636711, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5842567086219788, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01534179411828518, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.450087547302246, "metrics/particle/momentum/class_3/assigned/samples": 14536.0, "metrics/particle/momentum/class_3/efficiency": 0.7429924607276917, "metrics/particle/momentum/class_3/event/energy_bias": 5.759012222290039, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.22297477722168, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.992938041687012, "metrics/particle/momentum/class_3/event/px_bias": -0.015710026025772095, "metrics/particle/momentum/class_3/event/py_bias": 0.03907059505581856, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.699795722961426, "metrics/particle/momentum/class_3/event/pz_bias": -0.20997168123722076, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.180933952331543, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01280825026333332, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.058364395052194595, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.059201814234256744, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.014586899429559708, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.11104904115200043, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2675958573818207, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.09989257156848907, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8712942004203796, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10996338725090027, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7772974967956543, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.603630781173706, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.556379795074463, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.122098922729492, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.33408740162849426, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.11494109779596329, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0775163173675537, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -0.733401358127594, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.8519867658615112, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1827784776687622, "metrics/particle/momentum/class_4/event/px_bias": 0.11508224904537201, "metrics/particle/momentum/class_4/event/py_bias": 0.007746029645204544, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.205843210220337, "metrics/particle/momentum/class_4/event/pz_bias": 0.1399073600769043, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.392075538635254, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007384619675576687, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02102685160934925, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.020448952913284302, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006432292051613331, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.061413176357746124, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2146359384059906, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.16668933629989624, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8084232211112976, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.02687568962574005, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.02684403583407402, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.027550416067242622, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0042833369225263596, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09490740299224854, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4741480052471161, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4267283082008362, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5559604167938232, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.349215567111969, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9313242435455322, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.589442253112793, "metrics/particle/momentum/class_5/event/px_bias": 0.09569116681814194, "metrics/particle/momentum/class_5/event/py_bias": 0.052447106689214706, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6212668418884277, "metrics/particle/momentum/class_5/event/pz_bias": -0.21718309819698334, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4554853439331055, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.61940598487854, "step_valid_time": 9539.429901361465, "step_total_time": 9541.049307584763, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001943253882952, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06316731449578272, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9608623548922056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573694646397884, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9591127296805164, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042630535360211504, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9560475875743556, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9577884456215858, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9608623548922056, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003744194870519, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07293736911331816, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2134, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9471815357301376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9514043691484618, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.949288256227758, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04859563085153812, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9505127061970575, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9483985765124555, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9471815357301376, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066290016477044, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07419402183141244, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1182, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1178, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9198443579766536, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9227166276346604, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.921278254091972, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07728337236533958, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9167315175097276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9195940671350508, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9181605611847233, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9198443579766536} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_23000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_23000.json new file mode 100644 index 0000000000000000000000000000000000000000..b4d11553faca26ecc18ff82784e6af4a88810b67 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_23000.json @@ -0,0 +1 @@ +{"epoch": 1.089634261891226, "train": {"Classification_binary": 2.4237112998962402, "Classification": 0.01943134516477585, "Regression_pt": 0.10989486426115036, "Regression_eta": 0.010088645853102207, "Regression_sin_phi": 0.008543336763978004, "Regression_cos_phi": 0.009007848799228668, "Regression_energy": 0.11540170758962631, "Total": 8.071284294128418, "Total_unweighted": 2.6960790483281016}, "train_group": {}, "valid": {"Classification_binary": 2.504010041554769, "Classification": 0.020037269840637844, "Regression_pt": 0.1297279397646586, "Regression_eta": 0.010515786707401276, "Regression_sin_phi": 0.008219237749775251, "Regression_cos_phi": 0.008161749690771103, "Regression_energy": 0.13309034705162048, "Total": 8.665984471638998, "Total_unweighted": 2.8137623723596334, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02049030177295208, "diagnostic/loss/clic_pf/Classification_binary": 2.510140895843506, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007765496149659157, "diagnostic/loss/clic_pf/Regression_energy": 0.13517488539218903, "diagnostic/loss/clic_pf/Regression_eta": 0.00978908408433199, "diagnostic/loss/clic_pf/Regression_pt": 0.13215585052967072, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007789557799696922, "diagnostic/regression/clic_pf/energy_pred_mean": 0.033821333199739456, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1889488846063614, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04937495291233063, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42894527316093445, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0014197216369211674, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18264427781105042, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07026425004005432, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43066784739494324, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9694476127624512, "metrics/particle/class_1/pid_accuracy": 0.9875538349151611, "metrics/particle/class_2/efficiency": 0.4268155097961426, "metrics/particle/class_2/pid_accuracy": 0.744782030582428, "metrics/particle/class_3/efficiency": 0.750511884689331, "metrics/particle/class_3/pid_accuracy": 0.9863010048866272, "metrics/particle/class_4/efficiency": 0.9312015771865845, "metrics/particle/class_4/pid_accuracy": 0.4880332946777344, "metrics/particle/class_5/efficiency": 0.9939637780189514, "metrics/particle/class_5/pid_accuracy": 0.7267206311225891, "metrics/particle/count/bias_mean": -0.8235294222831726, "metrics/particle/count/mae": 3.22352933883667, "metrics/particle/count/prediction_mean": 69.17581939697266, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.046022940427064896, "metrics/particle/event/energy_response_mean": 1.0053282976150513, "metrics/particle/event/met_abs_error": 4.442485809326172, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03673906624317169, "metrics/particle/event/scalar_pt_response_mean": 1.0081223249435425, "metrics/particle/event/vector_pt_closure": 0.040407225489616394, "metrics/particle/matched/delta_eta_abs_mean": 0.008563965559005737, "metrics/particle/matched/delta_phi_abs_mean": 0.010675156489014626, "metrics/particle/matched/delta_r_mean": 0.015325675718486309, "metrics/particle/matched/energy_relative_abs_mean": 0.08279414474964142, "metrics/particle/matched/pid_accuracy": 0.969863772392273, "metrics/particle/matched/pt_relative_abs_mean": 0.07361739128828049, "metrics/particle/matching/duplicate_fraction": 0.026228517293930054, "metrics/particle/matching/efficiency": 0.8294194936752319, "metrics/particle/matching/f1": 0.8343273401260376, "metrics/particle/matching/purity": 0.8392936587333679, "metrics/particle/matching/target_coverage_dr0p05": 0.8035929203033447, "metrics/particle/matching/target_coverage_dr0p10": 0.8608670234680176, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7390732169151306, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8653115630149841, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017924806103110313, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07855647802352905, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07263799756765366, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01715800166130066, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12807618081569672, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003233256284147501, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3159840404987335, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.004268638789653778, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2172499895095825, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8660.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.27250170707702637, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8672142028808594, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9247217178344727, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8135603666305542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12021123617887497, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.333409309387207, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013826211914420128, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7190942168235779, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02098810113966465, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7651569843292236, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10415.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8056563138961792, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6263885498046875, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.863973617553711, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.6521482467651367, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.173136904835701, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.029322780668735504, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.023610591888428, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.11448238790035248, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.177064895629883, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026794608682394028, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09006375819444656, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0898151844739914, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03416265547275543, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14498700201511383, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016820400953292847, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.868642270565033, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.07020794600248337, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.894583225250244, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1843.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7203275561332703, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.379387617111206, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.311480760574341, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.1599156856536865, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.25096389651298523, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.300912857055664, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07045215368270874, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6789731979370117, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0861259400844574, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.638289451599121, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2853.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5999348759651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.39661648869514465, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.834527015686035, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0464553833007812, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.047498177736997604, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.057053305208683014, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.669749736785889, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.4058973491191864, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 12.592111587524414, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013273686170578003, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08410418778657913, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07264415174722672, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015395945869386196, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12892338633537292, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005739929038099945, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10263685137033463, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007786042522639036, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5687652230262756, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19164.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14137470722198486, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6461797952651978, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.47777295112609863, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3789408206939697, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08449392020702362, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.999468803405762, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028878848534077406, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2383127510547638, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.01195426844060421, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3776867389678955, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21469.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8729161024093628, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.4288175106048584, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.885408401489258, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.931910514831543, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.3047085702419281, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07561145722866058, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7624588012695312, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.2234424650669098, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.398221969604492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.015471339225769043, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0828549936413765, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07370907068252563, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017076143994927406, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12973442673683167, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023595241364091635, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21250082552433014, "metrics/particle/momentum/all/accepted/pz_bias": 0.0005780132487416267, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0779558420181274, "metrics/particle/momentum/all/accepted/samples": 29667.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2282399982213974, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7726708054542542, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7623841762542725, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6555244326591492, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10887525975704193, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 10.332964897155762, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011716613546013832, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5829176902770996, "metrics/particle/momentum/all/assigned/pz_bias": 0.0059781260788440704, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.873565673828125, "metrics/particle/momentum/all/assigned/samples": 34737.0, "metrics/particle/momentum/all/efficiency": 0.8292662501335144, "metrics/particle/momentum/all/event/energy_bias": -0.19904549419879913, "metrics/particle/momentum/all/event/p3_error_mean": 12.707858085632324, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.3596649169921875, "metrics/particle/momentum/all/event/px_bias": 0.4303472936153412, "metrics/particle/momentum/all/event/py_bias": 0.1619875580072403, "metrics/particle/momentum/all/event/pz_abs_mean": 9.535783767700195, "metrics/particle/momentum/all/event/pz_bias": -0.5148574113845825, "metrics/particle/momentum/all/event/pz_residual_rms": 15.004119873046875, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006971065420657396, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03800371661782265, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020796116441488266, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.006081368774175644, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.059822872281074524, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013464716030284762, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10811813920736313, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007896383292973042, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8296868205070496, "metrics/particle/momentum/class_1/accepted/samples": 16339.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.042911939322948456, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11288164556026459, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09422629326581955, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06870117038488388, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.019736841320991516, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.4983235597610474, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004385964944958687, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2186472862958908, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0004868767282459885, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.7610927820205688, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9683517813682556, "metrics/particle/momentum/class_1/event/energy_bias": 4.522317409515381, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.721888542175293, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.657014846801758, "metrics/particle/momentum/class_1/event/px_bias": -0.36019936203956604, "metrics/particle/momentum/class_1/event/py_bias": -0.2879806160926819, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.484881401062012, "metrics/particle/momentum/class_1/event/pz_bias": 0.6598606109619141, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.360086441040039, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037928368896245956, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19368165731430054, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1947430670261383, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.035573560744524, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23552994430065155, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02713487669825554, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6004952192306519, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.013211006298661232, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5854294300079346, "metrics/particle/momentum/class_2/accepted/samples": 1253.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.9230142831802368, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7554207444190979, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8500734567642212, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.33267879486083984, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.373236745595932, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.305663585662842, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07777354121208191, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.195574998855591, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.00846036896109581, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.687466621398926, "metrics/particle/momentum/class_2/assigned/samples": 2623.0, "metrics/particle/momentum/class_2/efficiency": 0.4253224730491638, "metrics/particle/momentum/class_2/event/energy_bias": -9.731078147888184, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.200514793395996, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.737201690673828, "metrics/particle/momentum/class_2/event/px_bias": 0.2348777949810028, "metrics/particle/momentum/class_2/event/py_bias": 0.05290772020816803, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.812980651855469, "metrics/particle/momentum/class_2/event/pz_bias": -0.9900580644607544, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.713626861572266, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02533702924847603, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13595205545425415, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13669104874134064, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03864818811416626, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17817752063274384, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012110726675018668, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20829829573631287, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.009656213223934174, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7625415921211243, "metrics/particle/momentum/class_3/accepted/samples": 11560.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3222719132900238, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5454456806182861, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5264874696731567, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3971644639968872, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16655327379703522, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 15.685370445251465, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008708667941391468, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5416854023933411, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.017554055899381638, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2726879119873047, "metrics/particle/momentum/class_3/assigned/samples": 14698.0, "metrics/particle/momentum/class_3/efficiency": 0.7500649094581604, "metrics/particle/momentum/class_3/event/energy_bias": 5.726532459259033, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.833172798156738, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.772624969482422, "metrics/particle/momentum/class_3/event/px_bias": 0.2771511673927307, "metrics/particle/momentum/class_3/event/py_bias": 0.25268614292144775, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.453673839569092, "metrics/particle/momentum/class_3/event/pz_bias": -0.09474672377109528, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.752728462219238, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011112156324088573, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05787383392453194, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05807781219482422, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.00985678844153881, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10684625804424286, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2719423174858093, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.004557038191705942, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8817955255508423, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.1129230260848999, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.6457816362380981, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.6125872135162354, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.5575650930404663, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.140941619873047, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3530852198600769, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02058263123035431, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.080701231956482, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.3650582432746887, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6701027154922485, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1251611709594727, "metrics/particle/momentum/class_4/event/px_bias": 0.2561531364917755, "metrics/particle/momentum/class_4/event/py_bias": 0.1067306399345398, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0233485698699951, "metrics/particle/momentum/class_4/event/pz_bias": 0.09369809925556183, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.418476104736328, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005217943340539932, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.0188930481672287, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.018232425674796104, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009627094492316246, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05332978069782257, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.18435075879096985, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.1024012416601181, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8669295310974121, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.031130369752645493, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.030334947630763054, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.029340835288167, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010146691463887691, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11247436702251434, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4851260483264923, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.40276142954826355, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.16526460647583, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.35175901651382446, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8740031123161316, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5610457062721252, "metrics/particle/momentum/class_5/event/px_bias": 0.022364526987075806, "metrics/particle/momentum/class_5/event/py_bias": 0.03764365613460541, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5726548433303833, "metrics/particle/momentum/class_5/event/pz_bias": -0.18361131846904755, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.390399217605591, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5601489543914795, "step_valid_time": 9972.904049158096, "step_total_time": 9974.464198589325, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001372855279582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06372718080871342, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2905, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2899, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9635157545605307, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9606481481481481, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.962079814538831, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03935185185185185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9615257048092869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9586640211640212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9600927305845338, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9635157545605307, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028553241805558, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07421682044451083, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2125, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9496659242761692, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9479768786127167, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05033407572383074, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9431868619618287, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9465478841870824, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9448643841707426, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9462938304482912, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066834292105495, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.072634237627901, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1190, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9260700389105059, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9282371294851794, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9271523178807947, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0717628705148206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9258970358814352, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9248149590962212, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9260700389105059} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_24000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_24000.json new file mode 100644 index 0000000000000000000000000000000000000000..14fe47f2d299fee33f0119253f43905f9eb3910d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_24000.json @@ -0,0 +1 @@ +{"epoch": 1.137009664582149, "train": {"Classification_binary": 2.4564995765686035, "Classification": 0.01993328332901001, "Regression_pt": 0.13409240543842316, "Regression_eta": 0.010314170271158218, "Regression_sin_phi": 0.008416744880378246, "Regression_cos_phi": 0.01064592320472002, "Regression_energy": 0.1351664662361145, "Total": 8.875624656677246, "Total_unweighted": 2.7750685699284077}, "train_group": {}, "valid": {"Classification_binary": 2.4881715774536133, "Classification": 0.01990179220835368, "Regression_pt": 0.12759599089622498, "Regression_eta": 0.010734771688779196, "Regression_sin_phi": 0.00819710890452067, "Regression_cos_phi": 0.008185970286528269, "Regression_energy": 0.13149450222651163, "Total": 8.601442337036133, "Total_unweighted": 2.7942817136645317, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020452380180358887, "diagnostic/loss/clic_pf/Classification_binary": 2.4993467330932617, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007845517247915268, "diagnostic/loss/clic_pf/Regression_energy": 0.13368980586528778, "diagnostic/loss/clic_pf/Regression_eta": 0.010076993145048618, "diagnostic/loss/clic_pf/Regression_pt": 0.12964874505996704, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007927787490189075, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02216777764260769, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18790124356746674, "diagnostic/regression/clic_pf/energy_residual_mean": 0.03772140294313431, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4212592542171478, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0049072070978581905, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18012666702270508, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06393732875585556, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42412081360816956, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9716739058494568, "metrics/particle/class_1/pid_accuracy": 0.9874595999717712, "metrics/particle/class_2/efficiency": 0.43155163526535034, "metrics/particle/class_2/pid_accuracy": 0.7548993825912476, "metrics/particle/class_3/efficiency": 0.7513093948364258, "metrics/particle/class_3/pid_accuracy": 0.9845942258834839, "metrics/particle/class_4/efficiency": 0.9399224519729614, "metrics/particle/class_4/pid_accuracy": 0.47422680258750916, "metrics/particle/class_5/efficiency": 0.9979879260063171, "metrics/particle/class_5/pid_accuracy": 0.7379032373428345, "metrics/particle/count/bias_mean": -0.08954248577356339, "metrics/particle/count/mae": 3.086928129196167, "metrics/particle/count/prediction_mean": 69.90980529785156, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04717005416750908, "metrics/particle/event/energy_response_mean": 0.996061384677887, "metrics/particle/event/met_abs_error": 4.533635139465332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0360771045088768, "metrics/particle/event/scalar_pt_response_mean": 1.0043514966964722, "metrics/particle/event/vector_pt_closure": 0.03856486454606056, "metrics/particle/matched/delta_eta_abs_mean": 0.010384959168732166, "metrics/particle/matched/delta_phi_abs_mean": 0.010757533833384514, "metrics/particle/matched/delta_r_mean": 0.016909921541810036, "metrics/particle/matched/energy_relative_abs_mean": 0.08425252884626389, "metrics/particle/matched/pid_accuracy": 0.9693593382835388, "metrics/particle/matched/pt_relative_abs_mean": 0.07263872027397156, "metrics/particle/matching/duplicate_fraction": 0.027187224477529526, "metrics/particle/matching/efficiency": 0.8313056230545044, "metrics/particle/matching/f1": 0.8318376541137695, "metrics/particle/matching/purity": 0.832370400428772, "metrics/particle/matching/target_coverage_dr0p05": 0.7960391640663147, "metrics/particle/matching/target_coverage_dr0p10": 0.862043559551239, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7450770139694214, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8709978461265564, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02086137980222702, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07910885661840439, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07049840688705444, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.00956239365041256, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12570078670978546, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0029009049758315086, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3152911961078644, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.023026922717690468, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1988672018051147, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8618.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.28202739357948303, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7623018622398376, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8282467722892761, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7094252705574036, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12180221825838089, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.218718528747559, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012695685029029846, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7108941078186035, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.026152784004807472, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.723482608795166, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10476.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8017489910125732, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -1.5977481603622437, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.953948974609375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5853068828582764, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4525755047798157, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.09069433808326721, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.142757892608643, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3688930869102478, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.233797073364258, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03183659911155701, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0905502438545227, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08705632388591766, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010626417584717274, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14456382393836975, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0210129301995039, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9587686061859131, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.017790760844945908, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.361976146697998, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1856.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7461727261543274, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.301135540008545, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 55034.5390625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 55034.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2517409324645996, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2949198.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0727715864777565, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.654529571533203, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.03300604224205017, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.035480499267578, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2872.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6041666865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.2036187648773193, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.970033645629883, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0972925424575806, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1494261920452118, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.019045064225792885, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.820478439331055, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.7170891761779785, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.78492546081543, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013907251879572868, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08640336990356445, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07277006655931473, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013436265289783478, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12932483851909637, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00041567080188542604, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1059982106089592, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.012010221369564533, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5680218935012817, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19246.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.14284799993038177, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6181871294975281, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4561786651611328, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3547845184803009, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0834876000881195, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.415977478027344, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024065161123871803, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23672588169574738, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.022031422704458237, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7595610618591309, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21608.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8766511678695679, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.46077436208724976, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.744071006774902, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.702658653259277, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.45662352442741394, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07550802826881409, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.805328130722046, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.36113688349723816, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.279191970825195, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01704344153404236, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08454712480306625, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07300351560115814, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01213747262954712, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1292964071035385, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024226109962910414, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21994264423847198, "metrics/particle/momentum/all/accepted/pz_bias": 0.015565760433673859, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1539381742477417, "metrics/particle/momentum/all/accepted/samples": 29720.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.23412823677062988, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7174883484840393, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4522.19189453125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4522.080078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10879391431808472, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 845347.5, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011271312832832336, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5774776339530945, "metrics/particle/momentum/all/assigned/pz_bias": 0.024168238043785095, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.0727429389953613, "metrics/particle/momentum/all/assigned/samples": 34956.0, "metrics/particle/momentum/all/efficiency": 0.8307477235794067, "metrics/particle/momentum/all/event/energy_bias": -3.340592622756958, "metrics/particle/momentum/all/event/p3_error_mean": 12.737473487854004, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.022256851196289, "metrics/particle/momentum/all/event/px_bias": 0.7597728967666626, "metrics/particle/momentum/all/event/py_bias": -0.03423137962818146, "metrics/particle/momentum/all/event/pz_abs_mean": 9.743958473205566, "metrics/particle/momentum/all/event/pz_bias": 1.709332823753357, "metrics/particle/momentum/all/event/pz_residual_rms": 17.67664337158203, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008566913194954395, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.041733112186193466, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01987439952790737, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0018459283746778965, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.058018941432237625, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013428553938865662, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.12071699649095535, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.008335883729159832, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9808271527290344, "metrics/particle/momentum/class_1/accepted/samples": 16383.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04057616740465164, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11739813536405563, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09408990293741226, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06048818677663803, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.018076216802001, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9648680686950684, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004207906313240528, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2312215268611908, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.015831992030143738, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.159132719039917, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9709595441818237, "metrics/particle/momentum/class_1/event/energy_bias": 0.7882112264633179, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.635507583618164, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.212376594543457, "metrics/particle/momentum/class_1/event/px_bias": -0.5071326494216919, "metrics/particle/momentum/class_1/event/py_bias": -0.2948317229747772, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.71132755279541, "metrics/particle/momentum/class_1/event/pz_bias": 0.35575729608535767, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.77353572845459, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03850230202078819, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19268642365932465, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1947171539068222, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03286132588982582, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23549789190292358, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02892884984612465, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6089457273483276, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03622452914714813, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5745484828948975, "metrics/particle/momentum/class_2/accepted/samples": 1279.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8909121751785278, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6449214220046997, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 58865.046875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 58864.54296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.36648043990135193, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3050169.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07411545515060425, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0111618041992188, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.0655430257320404, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.973062992095947, "metrics/particle/momentum/class_2/assigned/samples": 2685.0, "metrics/particle/momentum/class_2/efficiency": 0.43414798378944397, "metrics/particle/momentum/class_2/event/energy_bias": -8.620255470275879, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.976343154907227, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.466516494750977, "metrics/particle/momentum/class_2/event/px_bias": 0.5716750025749207, "metrics/particle/momentum/class_2/event/py_bias": 0.03704576566815376, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.744186401367188, "metrics/particle/momentum/class_2/event/pz_bias": 0.8596228361129761, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.858926773071289, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.027010688558220863, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13539475202560425, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13648846745491028, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.037540242075920105, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17832200229167938, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011270047398284078, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20697170495986938, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.024676397442817688, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.787125289440155, "metrics/particle/momentum/class_3/accepted/samples": 11535.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.34146302938461304, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.429207682609558, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.574709177017212, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4421231746673584, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16843947768211365, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.919931411743164, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008280597627162933, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5394624471664429, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.031051771715283394, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.8188467025756836, "metrics/particle/momentum/class_3/assigned/samples": 14854.0, "metrics/particle/momentum/class_3/efficiency": 0.7484427690505981, "metrics/particle/momentum/class_3/event/energy_bias": 5.517513275146484, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.68889331817627, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.743796348571777, "metrics/particle/momentum/class_3/event/px_bias": 0.6001296639442444, "metrics/particle/momentum/class_3/event/py_bias": 0.15917173027992249, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.310798168182373, "metrics/particle/momentum/class_3/event/pz_bias": 0.5017178058624268, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.8001070022583, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012336449697613716, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.049733567982912064, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.050091587007045746, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.002698968630284071, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09826573729515076, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22246712446212769, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.018300997093319893, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7847739458084106, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09445063769817352, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.36993104219436646, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.30058643221855164, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.24154937267303467, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03038674034178257, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.226984739303589, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.27821579575538635, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.00014938438835088164, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.978722333908081, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -0.7377846240997314, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5269670486450195, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9992649555206299, "metrics/particle/momentum/class_4/event/px_bias": 0.07461508363485336, "metrics/particle/momentum/class_4/event/py_bias": 0.014877919107675552, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9729725122451782, "metrics/particle/momentum/class_4/event/pz_bias": 0.09798745810985565, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.2761728763580322, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006339906249195337, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.020959515124559402, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.019300570711493492, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004936516750603914, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.060585517436265945, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.2079828530550003, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0617942214012146, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7933945059776306, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.00639190012589097, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0257299542427063, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.024072904139757156, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 3.1012754334369674e-05, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.08934081345796585, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4727945029735565, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.32740911841392517, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6741251945495605, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.2882768511772156, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9018298387527466, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5684418678283691, "metrics/particle/momentum/class_5/event/px_bias": 0.020485766232013702, "metrics/particle/momentum/class_5/event/py_bias": 0.04950496181845665, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5922194123268127, "metrics/particle/momentum/class_5/event/pz_bias": -0.10575248301029205, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.477050542831421, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5595362186431885, "step_valid_time": 10405.393751621246, "step_total_time": 10406.953288078308, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985850949732415, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06000114785938493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2880, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9537343027098479, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9554709485184573, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046265697290152015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552238805970149, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9517514871116987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534845224300612, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9572139303482587, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998659661553042, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07217735581867302, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2120, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2113, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9409675987572126, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9464285714285714, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.943690184731805, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05357142857142857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378606302707501, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9433035714285715, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9405742265746717, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9409675987572126, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000300906862694, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06756677879316053, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1181, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1177, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9190661478599221, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9204988308651598, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9197819314641744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07950116913484022, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9159533073929961, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9173811379579111, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9166666666666666, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9190661478599221} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_25000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_25000.json new file mode 100644 index 0000000000000000000000000000000000000000..fd39f66163399105cb140f6e84ea447b68cb1eee --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_25000.json @@ -0,0 +1 @@ +{"epoch": 1.1843850672730718, "train": {"Classification_binary": 2.492877960205078, "Classification": 0.019768817350268364, "Regression_pt": 0.11620720475912094, "Regression_eta": 0.009997026063501835, "Regression_sin_phi": 0.008466142229735851, "Regression_cos_phi": 0.00932153407484293, "Regression_energy": 0.11927180737257004, "Total": 8.315098762512207, "Total_unweighted": 2.775910492055118}, "train_group": {}, "valid": {"Classification_binary": 2.476310888926188, "Classification": 0.019835636019706726, "Regression_pt": 0.12790453433990479, "Regression_eta": 0.010517002393802008, "Regression_sin_phi": 0.008037636677424112, "Regression_cos_phi": 0.00808407242099444, "Regression_energy": 0.13108929991722107, "Total": 8.559247334798178, "Total_unweighted": 2.781779070695241, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02036026492714882, "diagnostic/loss/clic_pf/Classification_binary": 2.4896724224090576, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007631382439285517, "diagnostic/loss/clic_pf/Regression_energy": 0.1326552778482437, "diagnostic/loss/clic_pf/Regression_eta": 0.009818186983466148, "diagnostic/loss/clic_pf/Regression_pt": 0.1297149509191513, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.0075742932967841625, "diagnostic/regression/clic_pf/energy_pred_mean": 0.041463494300842285, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19120509922504425, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05701711028814316, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43400147557258606, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.011077131144702435, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.184500589966774, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0799216702580452, "diagnostic/regression/clic_pf/pt_residual_rms": 0.43697744607925415, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9688313603401184, "metrics/particle/class_1/pid_accuracy": 0.9871355295181274, "metrics/particle/class_2/efficiency": 0.4465493857860565, "metrics/particle/class_2/pid_accuracy": 0.7696969509124756, "metrics/particle/class_3/efficiency": 0.7432913184165955, "metrics/particle/class_3/pid_accuracy": 0.9847469925880432, "metrics/particle/class_4/efficiency": 0.9389534592628479, "metrics/particle/class_4/pid_accuracy": 0.5056759715080261, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7200811505317688, "metrics/particle/count/bias_mean": -1.1339869499206543, "metrics/particle/count/mae": 3.2790849208831787, "metrics/particle/count/prediction_mean": 68.8653564453125, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.052109599113464355, "metrics/particle/event/energy_response_mean": 1.021080493927002, "metrics/particle/event/met_abs_error": 4.76627254486084, "metrics/particle/event/scalar_pt_relative_abs_error": 0.041819341480731964, "metrics/particle/event/scalar_pt_response_mean": 1.0233410596847534, "metrics/particle/event/vector_pt_closure": 0.04106052964925766, "metrics/particle/matched/delta_eta_abs_mean": 0.00931417103856802, "metrics/particle/matched/delta_phi_abs_mean": 0.010449474677443504, "metrics/particle/matched/delta_r_mean": 0.015596435405313969, "metrics/particle/matched/energy_relative_abs_mean": 0.08352559804916382, "metrics/particle/matched/pid_accuracy": 0.9697448015213013, "metrics/particle/matched/pt_relative_abs_mean": 0.07345619052648544, "metrics/particle/matching/duplicate_fraction": 0.028017159551382065, "metrics/particle/matching/efficiency": 0.8277014493942261, "metrics/particle/matching/f1": 0.8344606161117554, "metrics/particle/matching/purity": 0.8413310050964355, "metrics/particle/matching/target_coverage_dr0p05": 0.8091391921043396, "metrics/particle/matching/target_coverage_dr0p10": 0.8613805770874023, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7362720370292664, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8627625107765198, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0178450308740139, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07973291724920273, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0734068900346756, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018400920554995537, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1300635188817978, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0033740547951310873, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3146807849407196, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.003480376675724983, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1920238733291626, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8595.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2654365003108978, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8771529793739319, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9262767434120178, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.820654034614563, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12046794593334198, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.60680103302002, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012665570713579655, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6954534649848938, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.009340488351881504, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.487560987472534, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10343.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7996092438697815, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7014563083648682, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.750200271606445, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.552732467651367, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.08098595589399338, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.006744943559169769, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.953378200531006, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.48472195863723755, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.545323371887207, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02556910365819931, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09042015671730042, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08819655328989029, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.027770472690463066, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.144108384847641, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021459227427840233, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9288725852966309, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.04110343009233475, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.065030336380005, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1864.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.672053337097168, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.553056001663208, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.676398754119873, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.542426347732544, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2459835708141327, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 56.42583084106445, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07104605436325073, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.565295457839966, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.019879162311553955, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.439571380615234, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2801.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6067708134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4557294547557831, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.619246482849121, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0828931331634521, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04347502812743187, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1629604995250702, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.447307109832764, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.9235075116157532, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.547643661499023, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013860632665455341, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08524485677480698, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07296939194202423, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017860811203718185, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13047410547733307, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006269592558965087, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10754787176847458, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0017854896141216159, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5658648610115051, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19140.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13468927145004272, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.8512321710586548, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.62884521484375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.5396186709403992, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08461789041757584, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 14.703545570373535, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029804871883243322, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2513931691646576, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01890539936721325, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.1340742111206055, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21473.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8718228936195374, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 3.158566951751709, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.470531463623047, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.317686080932617, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4539807438850403, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.5308635830879211, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.1888911724090576, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.627110481262207, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 11.07717227935791, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01575496979057789, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08397020399570465, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07405536621809006, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01864171028137207, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13125619292259216, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002736578928306699, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21941842138767242, "metrics/particle/momentum/all/accepted/pz_bias": -0.0047537037171423435, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1005817651748657, "metrics/particle/momentum/all/accepted/samples": 29599.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21723470091819763, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9157642126083374, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.964302659034729, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8665565848350525, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10838605463504791, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 20.476835250854492, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011381691321730614, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.571298360824585, "metrics/particle/momentum/all/assigned/pz_bias": 0.01290934532880783, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.025269031524658, "metrics/particle/momentum/all/assigned/samples": 34617.0, "metrics/particle/momentum/all/efficiency": 0.8273654580116272, "metrics/particle/momentum/all/event/energy_bias": 3.404294013977051, "metrics/particle/momentum/all/event/p3_error_mean": 13.588796615600586, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.413226127624512, "metrics/particle/momentum/all/event/px_bias": -0.4164698123931885, "metrics/particle/momentum/all/event/py_bias": -0.3746480345726013, "metrics/particle/momentum/all/event/pz_abs_mean": 10.439393997192383, "metrics/particle/momentum/all/event/pz_bias": 0.1883249580860138, "metrics/particle/momentum/all/event/pz_residual_rms": 20.220741271972656, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.008670898154377937, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.039041418582201004, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020429613068699837, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004304578062146902, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.060390904545784, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001287237973883748, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10758276283740997, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.007803210057318211, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.744339108467102, "metrics/particle/momentum/class_1/accepted/samples": 16314.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.04580157995223999, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11470341682434082, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08958645910024643, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.061863843351602554, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.021280379965901375, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.2998034954071045, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004327208269387484, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22389397025108337, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0079315435141325, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9831323623657227, "metrics/particle/momentum/class_1/assigned/samples": 16870.0, "metrics/particle/momentum/class_1/efficiency": 0.966870129108429, "metrics/particle/momentum/class_1/event/energy_bias": 2.9843907356262207, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.783202171325684, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.587773323059082, "metrics/particle/momentum/class_1/event/px_bias": -0.750058650970459, "metrics/particle/momentum/class_1/event/py_bias": -0.5256962776184082, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.604454040527344, "metrics/particle/momentum/class_1/event/pz_bias": 0.500898003578186, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.27118492126465, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038839880377054214, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19365766644477844, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1955796480178833, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02100089006125927, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23691457509994507, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03598200902342796, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.650239109992981, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.006243538577109575, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.800753355026245, "metrics/particle/momentum/class_2/accepted/samples": 1334.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8240565657615662, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6797123551368713, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8840096592903137, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.4429503083229065, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33180075883865356, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.498635768890381, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07318007946014404, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8871359825134277, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.019681235775351524, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.676451683044434, "metrics/particle/momentum/class_2/assigned/samples": 2610.0, "metrics/particle/momentum/class_2/efficiency": 0.4528173804283142, "metrics/particle/momentum/class_2/event/energy_bias": -4.159696578979492, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.62221622467041, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.761297225952148, "metrics/particle/momentum/class_2/event/px_bias": -0.11717398464679718, "metrics/particle/momentum/class_2/event/py_bias": -0.2954651117324829, "metrics/particle/momentum/class_2/event/pz_abs_mean": 10.238272666931152, "metrics/particle/momentum/class_2/event/pz_bias": -0.03151512145996094, "metrics/particle/momentum/class_2/event/pz_residual_rms": 16.942140579223633, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02338513731956482, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13712739944458008, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13778580725193024, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04429566487669945, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18021507561206818, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010496850591152906, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2117377072572708, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0004720148863270879, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7962817549705505, "metrics/particle/momentum/class_3/accepted/samples": 11432.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3114132881164551, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.905936598777771, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.015977382659912, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.8968511819839478, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17193174362182617, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 31.41718864440918, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008771328255534172, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.565385639667511, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.030021855607628822, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.9905834197998047, "metrics/particle/momentum/class_3/assigned/samples": 14593.0, "metrics/particle/momentum/class_3/efficiency": 0.7417596578598022, "metrics/particle/momentum/class_3/event/energy_bias": 5.433067321777344, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.608006477355957, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.670930862426758, "metrics/particle/momentum/class_3/event/px_bias": 0.3425883948802948, "metrics/particle/momentum/class_3/event/py_bias": 0.4525781273841858, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.2509541511535645, "metrics/particle/momentum/class_3/event/pz_bias": -0.28217098116874695, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.93816089630127, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.012868154793977737, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05586601793766022, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.056339021772146225, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.00497085228562355, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10453378409147263, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26829972863197327, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.03488596901297569, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8650722503662109, "metrics/particle/momentum/class_4/accepted/samples": 340.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11997807025909424, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.4776323437690735, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3812386393547058, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.32403475046157837, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0441988967359066, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.1734273433685303, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3252023458480835, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04758083447813988, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9896721243858337, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.939226508140564, "metrics/particle/momentum/class_4/event/energy_bias": -0.6103472113609314, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.4539413452148438, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9692543745040894, "metrics/particle/momentum/class_4/event/px_bias": 0.13914136588573456, "metrics/particle/momentum/class_4/event/py_bias": -0.011676102876663208, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8864502906799316, "metrics/particle/momentum/class_4/event/pz_bias": 0.09143048524856567, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.068974018096924, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0075299921445548534, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.019766245037317276, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01927495189011097, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008313772268593311, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05622856691479683, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.14656150341033936, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.08529072999954224, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4750753939151764, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.047622766345739365, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.031216854229569435, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030285950750112534, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0011243881890550256, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11399038136005402, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5259413719177246, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3101043403148651, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.237349987030029, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.24312056601047516, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8919527530670166, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5841765403747559, "metrics/particle/momentum/class_5/event/px_bias": -0.030967041850090027, "metrics/particle/momentum/class_5/event/py_bias": 0.005611341446638107, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5635407567024231, "metrics/particle/momentum/class_5/event/pz_bias": -0.09031738340854645, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4436538219451904, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6090023517608643, "step_valid_time": 10837.312675476074, "step_total_time": 10838.92167854309, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040247038745453, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059275893916860944, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2878, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540495867768595, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556291390728476, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045950413223140495, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9545605306799336, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514049586776859, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9529801324503311, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9572139303482587, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0093753750563659, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07566574863923214, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2129, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2122, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9449622725255216, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9504464285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9476964166481193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.049553571428571426, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.941855304039059, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9473214285714285, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9445804584909859, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9449622725255216, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0140929587442986, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07516544080699894, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1193, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9284046692607004, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9269619269619269, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9276827371695179, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07303807303807304, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9237354085603113, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9222999222999223, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9230171073094868, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9284046692607004} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_26000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_26000.json new file mode 100644 index 0000000000000000000000000000000000000000..13fdc33a028b35c6e9a801cc7280dec60c4ba7d1 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_26000.json @@ -0,0 +1 @@ +{"epoch": 1.2317604699639948, "train": {"Classification_binary": 2.480416774749756, "Classification": 0.021763574331998825, "Regression_pt": 0.1376539021730423, "Regression_eta": 0.01149385143071413, "Regression_sin_phi": 0.008592448197305202, "Regression_cos_phi": 0.010340587235987186, "Regression_energy": 0.14227166771888733, "Total": 9.17567253112793, "Total_unweighted": 2.812532805837691}, "train_group": {}, "valid": {"Classification_binary": 2.4780985514322915, "Classification": 0.019947797060012817, "Regression_pt": 0.12750847140947977, "Regression_eta": 0.010582509140173594, "Regression_sin_phi": 0.008055218805869421, "Regression_cos_phi": 0.008024783184130987, "Regression_energy": 0.13044285774230957, "Total": 8.55110232035319, "Total_unweighted": 2.7826601887742677, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020546356216073036, "diagnostic/loss/clic_pf/Classification_binary": 2.487118721008301, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007584099192172289, "diagnostic/loss/clic_pf/Regression_energy": 0.13329485058784485, "diagnostic/loss/clic_pf/Regression_eta": 0.010129306465387344, "diagnostic/loss/clic_pf/Regression_pt": 0.13042393326759338, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.00763061223551631, "diagnostic/regression/clic_pf/energy_pred_mean": 0.043706998229026794, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.19242767989635468, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05926061049103737, "diagnostic/regression/clic_pf/energy_residual_rms": 0.43004900217056274, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.00988996122032404, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18473921716213226, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07873449474573135, "diagnostic/regression/clic_pf/pt_residual_rms": 0.431481271982193, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9695668816566467, "metrics/particle/class_1/pid_accuracy": 0.9865302443504333, "metrics/particle/class_2/efficiency": 0.4386558532714844, "metrics/particle/class_2/pid_accuracy": 0.7588688731193542, "metrics/particle/class_3/efficiency": 0.738010585308075, "metrics/particle/class_3/pid_accuracy": 0.9846086502075195, "metrics/particle/class_4/efficiency": 0.930232584476471, "metrics/particle/class_4/pid_accuracy": 0.484375, "metrics/particle/class_5/efficiency": 0.9959758520126343, "metrics/particle/class_5/pid_accuracy": 0.7494949698448181, "metrics/particle/count/bias_mean": -1.2588235139846802, "metrics/particle/count/mae": 3.3019607067108154, "metrics/particle/count/prediction_mean": 68.74052429199219, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04809476062655449, "metrics/particle/event/energy_response_mean": 1.010573387145996, "metrics/particle/event/met_abs_error": 4.62224817276001, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03943542018532753, "metrics/particle/event/scalar_pt_response_mean": 1.0161361694335938, "metrics/particle/event/vector_pt_closure": 0.04137521982192993, "metrics/particle/matched/delta_eta_abs_mean": 0.01004349160939455, "metrics/particle/matched/delta_phi_abs_mean": 0.010103689506649971, "metrics/particle/matched/delta_r_mean": 0.015945857390761375, "metrics/particle/matched/energy_relative_abs_mean": 0.08476392179727554, "metrics/particle/matched/pid_accuracy": 0.96897953748703, "metrics/particle/matched/pt_relative_abs_mean": 0.07403675466775894, "metrics/particle/matching/duplicate_fraction": 0.028743118047714233, "metrics/particle/matching/efficiency": 0.8250404000282288, "metrics/particle/matching/f1": 0.8325262069702148, "metrics/particle/matching/purity": 0.8401491045951843, "metrics/particle/matching/target_coverage_dr0p05": 0.7996059656143188, "metrics/particle/matching/target_coverage_dr0p10": 0.8594757914543152, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7335829734802246, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8609043955802917, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.018345752730965614, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08161904662847519, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07418815791606903, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.021339351311326027, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1315644532442093, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003144653979688883, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32173842191696167, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.025443032383918762, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2035682201385498, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8586.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2680126428604126, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9230964183807373, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9865351915359497, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8840257525444031, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11997675895690918, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.367426872253418, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01220102608203888, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6975142955780029, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.00313006155192852, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.9689040184020996, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10327.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7987719774246216, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.120321273803711, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 6.1927337646484375, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.8139660358428955, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.27633336186408997, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.2236047238111496, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.263473033905029, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.15739449858665466, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.655496597290039, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.028060700744390488, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08826906979084015, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08809467405080795, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023343555629253387, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1453665792942047, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.025205479934811592, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9341541528701782, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.15432560443878174, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.189246892929077, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1825.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7285023927688599, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.683125376701355, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.703263998031616, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.5580618381500244, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24045665562152863, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 19.092430114746094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07563325017690659, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.935947895050049, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.0852096751332283, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.935046195983887, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2803.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5940755009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.235375165939331, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.759069442749023, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0962458848953247, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.107050821185112, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.08908943831920624, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.596755504608154, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3732435405254364, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.682119369506836, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.013844083063304424, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08577048778533936, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07270915061235428, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.02048536203801632, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12971310317516327, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005769432755187154, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10779061168432236, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.012068213894963264, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5396180152893066, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19066.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13810084760189056, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7924832701683044, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.581768274307251, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.49455565214157104, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08875849843025208, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.8427534103393555, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0033063238952308893, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2576923072338104, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0029304814524948597, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.8532156944274902, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21474.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8684521913528442, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.614600658416748, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.228399753570557, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 5.136108875274658, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.5758174061775208, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2780326008796692, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.042226552963257, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.042493149638175964, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.48692798614502, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01603551022708416, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08471596986055374, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07409250736236572, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020911067724227905, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13127443194389343, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0028496794402599335, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22127126157283783, "metrics/particle/momentum/all/accepted/pz_bias": 0.024771537631750107, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1135588884353638, "metrics/particle/momentum/all/accepted/samples": 29477.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22469474375247955, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9036065936088562, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8744101524353027, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7779351472854614, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1103629618883133, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 9.521583557128906, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011819442734122276, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6058946251869202, "metrics/particle/momentum/all/assigned/pz_bias": 0.006017735693603754, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.5717103481292725, "metrics/particle/momentum/all/assigned/samples": 34604.0, "metrics/particle/momentum/all/efficiency": 0.8239552974700928, "metrics/particle/momentum/all/event/energy_bias": 0.4995466470718384, "metrics/particle/momentum/all/event/p3_error_mean": 13.694804191589355, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.773910045623779, "metrics/particle/momentum/all/event/px_bias": -0.40653491020202637, "metrics/particle/momentum/all/event/py_bias": -0.5907267332077026, "metrics/particle/momentum/all/event/pz_abs_mean": 10.361732482910156, "metrics/particle/momentum/all/event/pz_bias": 0.2583422064781189, "metrics/particle/momentum/all/event/pz_residual_rms": 18.508872985839844, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007740756496787071, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04001156613230705, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020667914301156998, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0026824160013347864, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06075793504714966, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014697776641696692, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11161264777183533, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.025471121072769165, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8317471146583557, "metrics/particle/momentum/class_1/accepted/samples": 16329.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.042472660541534424, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10182202607393265, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.078124038875103, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.04974109306931496, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020685158669948578, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.105921745300293, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0043266951106488705, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22598204016685486, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03337647020816803, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.1050074100494385, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9677591323852539, "metrics/particle/momentum/class_1/event/energy_bias": 2.106261730194092, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.785079002380371, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.620458126068115, "metrics/particle/momentum/class_1/event/px_bias": -0.45093968510627747, "metrics/particle/momentum/class_1/event/py_bias": -0.16383221745491028, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.572380065917969, "metrics/particle/momentum/class_1/event/pz_bias": 1.020729899406433, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.116040229797363, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.038502760231494904, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19154201447963715, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19345711171627045, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02081470750272274, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23431311547756195, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03379416465759277, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6431567668914795, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07964883744716644, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.745739698410034, "metrics/particle/momentum/class_2/accepted/samples": 1302.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.854131817817688, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8777363896369934, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9626297354698181, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.5075569748878479, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3488820493221283, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.403013229370117, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07363145798444748, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.217833995819092, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.04530217498540878, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.504775047302246, "metrics/particle/momentum/class_2/assigned/samples": 2594.0, "metrics/particle/momentum/class_2/efficiency": 0.44195517897605896, "metrics/particle/momentum/class_2/event/energy_bias": -6.752392768859863, "metrics/particle/momentum/class_2/event/p3_error_mean": 13.358796119689941, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.924805164337158, "metrics/particle/momentum/class_2/event/px_bias": 0.20647035539150238, "metrics/particle/momentum/class_2/event/py_bias": -0.6807612180709839, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.867827415466309, "metrics/particle/momentum/class_2/event/pz_bias": 0.3796837627887726, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.778156280517578, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.025726402178406715, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13884390890598297, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13887380063533783, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.052655257284641266, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18123258650302887, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0014121800195425749, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21630971133708954, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.017910564318299294, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7677205801010132, "metrics/particle/momentum/class_3/accepted/samples": 11330.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3289886713027954, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8345309495925903, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7787971496582031, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6653831005096436, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17438673973083496, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.167091369628906, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009798547253012657, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5891949534416199, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.012752989307045937, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.467108726501465, "metrics/particle/momentum/class_3/assigned/samples": 14594.0, "metrics/particle/momentum/class_3/efficiency": 0.7351414561271667, "metrics/particle/momentum/class_3/event/energy_bias": 6.138795852661133, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.424454689025879, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.102791786193848, "metrics/particle/momentum/class_3/event/px_bias": -0.36868351697921753, "metrics/particle/momentum/class_3/event/py_bias": 0.06175542622804642, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.866752624511719, "metrics/particle/momentum/class_3/event/pz_bias": -1.1499603986740112, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.715849876403809, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01052042469382286, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05108349770307541, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05151905491948128, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004993606358766556, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09897437691688538, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22153334319591522, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.027841590344905853, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8141691088676453, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09827334433794022, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.3502252101898193, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.307986855506897, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2541441917419434, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 17.567676544189453, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.31464922428131104, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.004647514317184687, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1840628385543823, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.664749801158905, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6685165166854858, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.1062610149383545, "metrics/particle/momentum/class_4/event/px_bias": 0.130541130900383, "metrics/particle/momentum/class_4/event/py_bias": 0.0898064523935318, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.0271317958831787, "metrics/particle/momentum/class_4/event/pz_bias": 0.11719100922346115, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8366379737854004, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.006301569752395153, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.023168396204710007, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02170436643064022, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007962215691804886, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06608858704566956, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.19597646594047546, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.00950862281024456, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6954344511032104, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03452586755156517, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06420426815748215, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05308881774544716, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.02874971181154251, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.367389976978302, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5161117911338806, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.29088711738586426, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.140129566192627, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.328368604183197, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.943865180015564, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6172215938568115, "metrics/particle/momentum/class_5/event/px_bias": 0.07607684284448624, "metrics/particle/momentum/class_5/event/py_bias": 0.10230474174022675, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5936539173126221, "metrics/particle/momentum/class_5/event/pz_bias": -0.10930220782756805, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.421539783477783, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6684238910675049, "step_valid_time": 11271.250836849213, "step_total_time": 11272.919261217117, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013085154786732, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0615285003349888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3033, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9538410814375206, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566798941798942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0461589185624794, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956882255389718, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9512034289482361, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9540343915343915, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9595356550580432, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0065557366144415, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07218752052164157, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2235, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2122, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2116, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.941855304039059, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9494407158836689, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456327985739751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.050559284116331094, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9391921881935198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9467561521252796, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9429590017825312, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.941855304039059, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0105291785121415, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07430742128942847, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1280, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1172, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9151750972762646, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.91875, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9169590643274854, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9120622568093385, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.915625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9138401559454191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9151750972762646} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_27000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_27000.json new file mode 100644 index 0000000000000000000000000000000000000000..e0956a38e27f13251f9bdd01ee3a580bdaec091c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_27000.json @@ -0,0 +1 @@ +{"epoch": 1.2791358726549176, "train": {"Classification_binary": 2.5996758937835693, "Classification": 0.02137037366628647, "Regression_pt": 0.14069430530071259, "Regression_eta": 0.01524403877556324, "Regression_sin_phi": 0.008781380951404572, "Regression_cos_phi": 0.010478058829903603, "Regression_energy": 0.14581483602523804, "Total": 9.577415466308594, "Total_unweighted": 2.942058887332678}, "train_group": {}, "valid": {"Classification_binary": 2.4496165911356607, "Classification": 0.019676821927229565, "Regression_pt": 0.12606390317281088, "Regression_eta": 0.010316142191489538, "Regression_sin_phi": 0.008055893704295158, "Regression_cos_phi": 0.008028077582518259, "Regression_energy": 0.1293913722038269, "Total": 8.462164560953775, "Total_unweighted": 2.751148801917831, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020091194659471512, "diagnostic/loss/clic_pf/Classification_binary": 2.4527781009674072, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007609768770635128, "diagnostic/loss/clic_pf/Regression_energy": 0.13208724558353424, "diagnostic/loss/clic_pf/Regression_eta": 0.009643394500017166, "diagnostic/loss/clic_pf/Regression_pt": 0.12876224517822266, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007651217747479677, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03189157694578171, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18719585239887238, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04744520038366318, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4217769503593445, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0008700868347659707, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.18022562563419342, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06797444820404053, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42265602946281433, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9713360071182251, "metrics/particle/class_1/pid_accuracy": 0.987230122089386, "metrics/particle/class_2/efficiency": 0.4376409649848938, "metrics/particle/class_2/pid_accuracy": 0.7570213675498962, "metrics/particle/class_3/efficiency": 0.7459855675697327, "metrics/particle/class_3/pid_accuracy": 0.9856399893760681, "metrics/particle/class_4/efficiency": 0.9370155334472656, "metrics/particle/class_4/pid_accuracy": 0.5232678651809692, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7545638680458069, "metrics/particle/count/bias_mean": -1.384313702583313, "metrics/particle/count/mae": 3.3111112117767334, "metrics/particle/count/prediction_mean": 68.61503601074219, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04546311870217323, "metrics/particle/event/energy_response_mean": 0.9988212585449219, "metrics/particle/event/met_abs_error": 4.316184043884277, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03520284965634346, "metrics/particle/event/scalar_pt_response_mean": 1.0042619705200195, "metrics/particle/event/vector_pt_closure": 0.037954553961753845, "metrics/particle/matched/delta_eta_abs_mean": 0.008075345307588577, "metrics/particle/matched/delta_phi_abs_mean": 0.009336446411907673, "metrics/particle/matched/delta_r_mean": 0.013879982754588127, "metrics/particle/matched/energy_relative_abs_mean": 0.08298788964748383, "metrics/particle/matched/pid_accuracy": 0.9702080488204956, "metrics/particle/matched/pt_relative_abs_mean": 0.07310140132904053, "metrics/particle/matching/duplicate_fraction": 0.025928501039743423, "metrics/particle/matching/efficiency": 0.8292887806892395, "metrics/particle/matching/f1": 0.8375707268714905, "metrics/particle/matching/purity": 0.8460197448730469, "metrics/particle/matching/target_coverage_dr0p05": 0.8074211478233337, "metrics/particle/matching/target_coverage_dr0p10": 0.8610164523124695, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7396707534790039, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8648446798324585, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01619976945221424, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.077762670814991, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07146333903074265, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015999021008610725, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12686841189861298, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030148422811180353, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3067992627620697, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.007878798991441727, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.163910984992981, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8624.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25854748487472534, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8066065311431885, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8448402285575867, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7376012206077576, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11754471063613892, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.866209506988525, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011889802291989326, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6583125591278076, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0048016151413321495, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.349106550216675, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10345.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8023071885108948, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.47209423780441284, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.738502502441406, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5250494480133057, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.23926770687103271, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.004080059006810188, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.96433162689209, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.18874284625053406, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.257282257080078, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.026409465819597244, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08479655534029007, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08470871299505234, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023826517164707184, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13897982239723206, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018378378823399544, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8379240036010742, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05267612636089325, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6390321254730225, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1850.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6829146146774292, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.195286750793457, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.3624401092529297, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.2188353538513184, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2345590889453888, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.38068389892578, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06747590005397797, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5422041416168213, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.08255773782730103, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.032646179199219, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2801.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6022135615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.7211568355560303, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.294227600097656, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.007367730140686, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.00046187639236450195, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08039969950914383, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.160395622253418, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.2790789604187012, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.735617637634277, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011878910474479198, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08505548536777496, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07275093346834183, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015786122530698776, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12973244488239288, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006777540547773242, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10378469526767731, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006249607540667057, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5726962685585022, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19181.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13433195650577545, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5279750823974609, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4053628146648407, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.31196191906929016, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08295274525880814, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.797281265258789, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002143722726032138, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22012773156166077, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0007946107652969658, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4066345691680908, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21458.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8736904263496399, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.33308693766593933, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.372706890106201, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.400341987609863, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.4397684335708618, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.13356587290763855, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.6392478942871094, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.06506551802158356, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.3747053146362305, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014041938818991184, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08291850239038467, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07312245666980743, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016349628567695618, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12950599193572998, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024616422597318888, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20862217247486115, "metrics/particle/momentum/all/accepted/pz_bias": 0.0015351163456216455, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.020084023475647, "metrics/particle/momentum/all/accepted/samples": 29655.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21587136387825012, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6652881503105164, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6951608061790466, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5935591459274292, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10556583106517792, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 7.401819705963135, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010345624759793282, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5390838384628296, "metrics/particle/momentum/all/assigned/pz_bias": -0.008610786870121956, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.845957040786743, "metrics/particle/momentum/all/assigned/samples": 34604.0, "metrics/particle/momentum/all/efficiency": 0.8289307951927185, "metrics/particle/momentum/all/event/energy_bias": -2.860164165496826, "metrics/particle/momentum/all/event/p3_error_mean": 12.506965637207031, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.839326858520508, "metrics/particle/momentum/all/event/px_bias": 0.6785742044448853, "metrics/particle/momentum/all/event/py_bias": -0.05724623426795006, "metrics/particle/momentum/all/event/pz_abs_mean": 9.706527709960938, "metrics/particle/momentum/all/event/pz_bias": -1.155401587486267, "metrics/particle/momentum/all/event/pz_residual_rms": 17.525001525878906, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006500540301203728, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03886140510439873, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020833652466535568, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004393641371279955, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06054244935512543, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014041514368727803, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10628549009561539, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005410709884017706, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7933849096298218, "metrics/particle/momentum/class_1/accepted/samples": 16380.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.035892732441425323, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.08468977361917496, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07486513257026672, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.048030585050582886, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.01789841800928116, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6152544021606445, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004148639738559723, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.21530187129974365, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0036443094722926617, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.2363767623901367, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9707817435264587, "metrics/particle/momentum/class_1/event/energy_bias": 1.6731486320495605, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.881043434143066, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.54664945602417, "metrics/particle/momentum/class_1/event/px_bias": -0.5151849985122681, "metrics/particle/momentum/class_1/event/py_bias": -0.116801917552948, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.772319793701172, "metrics/particle/momentum/class_1/event/pz_bias": 0.13965044915676117, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.230463981628418, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03703777492046356, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1959415227174759, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19745399057865143, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022447573021054268, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2395767718553543, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.0315384604036808, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5450985431671143, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03549287095665932, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3436036109924316, "metrics/particle/momentum/class_2/accepted/samples": 1300.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8472262024879456, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6202472448348999, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7841174602508545, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.31517133116722107, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34754225611686707, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.803622245788574, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06950844824314117, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.898526906967163, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0716891661286354, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.036999225616455, "metrics/particle/momentum/class_2/assigned/samples": 2604.0, "metrics/particle/momentum/class_2/efficiency": 0.44127631187438965, "metrics/particle/momentum/class_2/event/energy_bias": -8.41604995727539, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.648420333862305, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.469749450683594, "metrics/particle/momentum/class_2/event/px_bias": 0.9331993460655212, "metrics/particle/momentum/class_2/event/py_bias": -0.08809538930654526, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.376920700073242, "metrics/particle/momentum/class_2/event/pz_bias": -1.3537437915802002, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.146190643310547, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022433236241340637, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13485153019428253, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1350967437028885, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03809511289000511, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17696014046669006, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007855459698475897, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20260044932365417, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006702867802232504, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7043882012367249, "metrics/particle/momentum/class_3/accepted/samples": 11457.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3175298273563385, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3556116819381714, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4146480560302734, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2897409200668335, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16615237295627594, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.149670600891113, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007268737535923719, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49931278824806213, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.007568384986370802, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.0996601581573486, "metrics/particle/momentum/class_3/assigned/samples": 14583.0, "metrics/particle/momentum/class_3/efficiency": 0.7433817982673645, "metrics/particle/momentum/class_3/event/energy_bias": 4.325213432312012, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.63638687133789, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.755152225494385, "metrics/particle/momentum/class_3/event/px_bias": 0.03220108151435852, "metrics/particle/momentum/class_3/event/py_bias": 0.17611123621463776, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.194277763366699, "metrics/particle/momentum/class_3/event/pz_bias": -0.09396570920944214, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.510722160339355, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011825758963823318, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.057576488703489304, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.057707708328962326, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.012409703806042671, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10249070823192596, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26378536224365234, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.004201046656817198, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8412337899208069, "metrics/particle/momentum/class_4/accepted/samples": 340.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07418716698884964, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.560213565826416, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3166716396808624, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.26977989077568054, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 1.8662614822387695, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3075847625732422, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.008814304135739803, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9165970087051392, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.939226508140564, "metrics/particle/momentum/class_4/event/energy_bias": -0.17911553382873535, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.6718757152557373, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.027011513710022, "metrics/particle/momentum/class_4/event/px_bias": 0.12696820497512817, "metrics/particle/momentum/class_4/event/py_bias": -0.06980165839195251, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.139299988746643, "metrics/particle/momentum/class_4/event/pz_bias": 0.33024969696998596, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.537529468536377, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004198878537863493, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01744074374437332, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01728762872517109, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007793402299284935, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04537303000688553, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1473356932401657, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06191672757267952, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7065566182136536, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.004530421923846006, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03217291459441185, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.032157618552446365, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013425620272755623, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11446449160575867, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4454968571662903, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3604426980018616, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.5880014896392822, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.26336103677749634, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8456284999847412, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5818682909011841, "metrics/particle/momentum/class_5/event/px_bias": 0.10139055550098419, "metrics/particle/momentum/class_5/event/py_bias": 0.041341476142406464, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5111268758773804, "metrics/particle/momentum/class_5/event/pz_bias": -0.17759230732917786, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.286223888397217, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5559473037719727, "step_valid_time": 11706.26890039444, "step_total_time": 11707.824848175049, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0026776459755997, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05936736905527584, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2888, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579748510919921, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9590856385621997, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.042025148908007945, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9578772802653399, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9556585043017869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9567666059300978, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9601990049751243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015320826549798, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06887314209625384, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9535714285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9508123748052526, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04642857142857143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467376830892144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522321428571429, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494769641664812, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948069241011984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002459875698945, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06709046239006655, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1192, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1190, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9305230288836847, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9290724863600935, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06947697111631537, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9260700389105059, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9289617486338798, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9275136399064692, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9276264591439689} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_28000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_28000.json new file mode 100644 index 0000000000000000000000000000000000000000..815959faae2e94f6fbb004603adf8850519ea8af --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_28000.json @@ -0,0 +1 @@ +{"epoch": 1.3265112753458403, "train": {"Classification_binary": 2.4223363399505615, "Classification": 0.02020343206822872, "Regression_pt": 0.12992577254772186, "Regression_eta": 0.009449491277337074, "Regression_sin_phi": 0.007941417396068573, "Regression_cos_phi": 0.007483202498406172, "Regression_energy": 0.12949050962924957, "Total": 8.43094253540039, "Total_unweighted": 2.7268301653675735}, "train_group": {}, "valid": {"Classification_binary": 2.4363581339518228, "Classification": 0.019528763989607494, "Regression_pt": 0.12538041671117148, "Regression_eta": 0.010279298449556032, "Regression_sin_phi": 0.007984168827533722, "Regression_cos_phi": 0.008051765461762747, "Regression_energy": 0.12877376874287924, "Total": 8.41955820719401, "Total_unweighted": 2.7363563161343336, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020065482705831528, "diagnostic/loss/clic_pf/Classification_binary": 2.4382009506225586, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.0075814202427864075, "diagnostic/loss/clic_pf/Regression_energy": 0.1326020210981369, "diagnostic/loss/clic_pf/Regression_eta": 0.009587304666638374, "diagnostic/loss/clic_pf/Regression_pt": 0.1291140615940094, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007610625587403774, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03088473156094551, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1870546191930771, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04643835499882698, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4240538477897644, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.004687740001827478, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1791788637638092, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06415680050849915, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4246354401111603, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9713956117630005, "metrics/particle/class_1/pid_accuracy": 0.9871081113815308, "metrics/particle/class_2/efficiency": 0.4459855556488037, "metrics/particle/class_2/pid_accuracy": 0.768394410610199, "metrics/particle/class_3/efficiency": 0.759909451007843, "metrics/particle/class_3/pid_accuracy": 0.9857896566390991, "metrics/particle/class_4/efficiency": 0.9370155334472656, "metrics/particle/class_4/pid_accuracy": 0.5377455949783325, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7383367419242859, "metrics/particle/count/bias_mean": -0.6183006763458252, "metrics/particle/count/mae": 3.1032679080963135, "metrics/particle/count/prediction_mean": 69.38104248046875, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04445867985486984, "metrics/particle/event/energy_response_mean": 1.006386399269104, "metrics/particle/event/met_abs_error": 4.257333278656006, "metrics/particle/event/scalar_pt_relative_abs_error": 0.034790635108947754, "metrics/particle/event/scalar_pt_response_mean": 1.0066988468170166, "metrics/particle/event/vector_pt_closure": 0.03712435066699982, "metrics/particle/matched/delta_eta_abs_mean": 0.009024210274219513, "metrics/particle/matched/delta_phi_abs_mean": 0.009164458140730858, "metrics/particle/matched/delta_r_mean": 0.014526739716529846, "metrics/particle/matched/energy_relative_abs_mean": 0.08345146477222443, "metrics/particle/matched/pid_accuracy": 0.970705509185791, "metrics/particle/matched/pt_relative_abs_mean": 0.07264650613069534, "metrics/particle/matching/duplicate_fraction": 0.026254557073116302, "metrics/particle/matching/efficiency": 0.8360395431518555, "metrics/particle/matching/f1": 0.8397482633590698, "metrics/particle/matching/purity": 0.8434900641441345, "metrics/particle/matching/target_coverage_dr0p05": 0.8156845569610596, "metrics/particle/matching/target_coverage_dr0p10": 0.8669642210006714, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7470564842224121, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8716140985488892, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016117267310619354, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08023334294557571, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07248537242412567, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013686486519873142, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12787708640098572, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0029833619482815266, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3036405146121979, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.012185017578303814, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.14934504032135, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8715.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2671763002872467, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8571540713310242, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8198341131210327, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7080301642417908, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11522239446640015, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.5054192543029785, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013036809861660004, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6760616302490234, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07262997329235077, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.42393159866333, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10432.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8107730746269226, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.036679089069366455, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.527815818786621, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.412496328353882, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.10017363727092743, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.10666011273860931, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.796757698059082, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.6182636618614197, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.724954605102539, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024342158809304237, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08989237993955612, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08614453673362732, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013628179207444191, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14085930585861206, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020386826246976852, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9186116456985474, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06032130867242813, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.015040636062622, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1913.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.657562255859375, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1410338878631592, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.2608726024627686, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.113095760345459, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2288016825914383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 18.454204559326172, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06061667948961258, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.498746633529663, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.12108336389064789, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.980859279632568, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2854.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6227213740348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7567487955093384, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.356893062591553, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.968661904335022, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.056484729051589966, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.068858802318573, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.227550029754639, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.47928643226623535, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.942803382873535, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012929956428706646, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08481680601835251, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07197180390357971, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011330698616802692, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12810637056827545, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000519022170919925, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10376311838626862, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.008404090069234371, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5572701096534729, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19267.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13706426322460175, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5917787551879883, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4354664385318756, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33522823452949524, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08245556801557541, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.838676452636719, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0025984873063862324, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2306544929742813, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.01838010363280773, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4391556978225708, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21551.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8776077032089233, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.13601627945899963, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.373663425445557, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.429792404174805, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.13524918258190155, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.39147090911865234, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.571544885635376, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.4395807981491089, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.930927276611328, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014589397236704826, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08380542695522308, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07302844524383545, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01216447725892067, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12889385223388672, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002508780686184764, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21417412161827087, "metrics/particle/momentum/all/accepted/pz_bias": 0.012828522361814976, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0802534818649292, "metrics/particle/momentum/all/accepted/samples": 29895.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21866799890995026, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7162432670593262, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7001112699508667, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5925151109695435, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10425696521997452, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.11526107788086, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010477365925908089, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5498444437980652, "metrics/particle/momentum/all/assigned/pz_bias": 0.0430392250418663, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8738107681274414, "metrics/particle/momentum/all/assigned/samples": 34837.0, "metrics/particle/momentum/all/efficiency": 0.8356394171714783, "metrics/particle/momentum/all/event/energy_bias": -0.6574115753173828, "metrics/particle/momentum/all/event/p3_error_mean": 12.160527229309082, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.693262577056885, "metrics/particle/momentum/all/event/px_bias": -0.09156027436256409, "metrics/particle/momentum/all/event/py_bias": -0.21595197916030884, "metrics/particle/momentum/all/event/pz_abs_mean": 9.323486328125, "metrics/particle/momentum/all/event/pz_bias": 1.5371310710906982, "metrics/particle/momentum/all/event/pz_residual_rms": 17.697607040405273, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006817850284278393, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03884803503751755, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02007378078997135, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003509484464302659, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05901589244604111, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014652909012511373, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10407979041337967, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0064859637059271336, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8266760110855103, "metrics/particle/momentum/class_1/accepted/samples": 16379.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.036348361521959305, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.102871835231781, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08416079729795456, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.057621002197265625, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.018136557191610336, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7596877813339233, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089615773409605, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2150908261537552, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03417983651161194, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3352303504943848, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9707224369049072, "metrics/particle/momentum/class_1/event/energy_bias": 2.74662184715271, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.423595428466797, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.415294647216797, "metrics/particle/momentum/class_1/event/px_bias": -0.6420652866363525, "metrics/particle/momentum/class_1/event/py_bias": -0.5947384834289551, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.282229423522949, "metrics/particle/momentum/class_1/event/pz_bias": 1.3637466430664062, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.652850151062012, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03790999576449394, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19572371244430542, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19573289155960083, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03385048359632492, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2359694242477417, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029770992696285248, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6254777908325195, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.22434137761592865, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.556617021560669, "metrics/particle/momentum/class_2/accepted/samples": 1310.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8460997939109802, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6615918278694153, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8172817826271057, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3338385820388794, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.35752078890800476, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.547450542449951, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06727135181427002, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9557883739471436, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.12766143679618835, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.152291774749756, "metrics/particle/momentum/class_2/assigned/samples": 2646.0, "metrics/particle/momentum/class_2/efficiency": 0.44467073678970337, "metrics/particle/momentum/class_2/event/energy_bias": -7.314146041870117, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.467034339904785, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.390196800231934, "metrics/particle/momentum/class_2/event/px_bias": -0.030290300026535988, "metrics/particle/momentum/class_2/event/py_bias": -0.12458576261997223, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.241765022277832, "metrics/particle/momentum/class_2/event/pz_bias": 0.5429493188858032, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.053207397460938, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023114120587706566, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13605156540870667, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13479255139827728, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02964373305439949, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17607688903808594, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010266940807923675, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2105897068977356, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.001083390205167234, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.764321506023407, "metrics/particle/momentum/class_3/accepted/samples": 11688.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3201775550842285, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3925657272338867, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3537745475769043, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.218434453010559, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15966159105300903, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 10.967168807983398, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007851099595427513, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5075255036354065, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04394939914345741, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.959386944770813, "metrics/particle/momentum/class_3/assigned/samples": 14775.0, "metrics/particle/momentum/class_3/efficiency": 0.7583701014518738, "metrics/particle/momentum/class_3/event/energy_bias": 4.937715530395508, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.798696517944336, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.803860664367676, "metrics/particle/momentum/class_3/event/px_bias": 0.3981228470802307, "metrics/particle/momentum/class_3/event/py_bias": 0.4157561659812927, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.401212692260742, "metrics/particle/momentum/class_3/event/pz_bias": -0.33251065015792847, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.724529266357422, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01089149434119463, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05699015036225319, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05739700794219971, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007079864386469126, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10778756439685822, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2362561821937561, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0048383306711912155, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7971352338790894, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08133041858673096, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.445775032043457, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.2109596729278564, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1604807376861572, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 33.662025451660156, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3095206022262573, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.02175384946167469, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.024872064590454, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -0.667705774307251, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.384124994277954, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9263637661933899, "metrics/particle/momentum/class_4/event/px_bias": 0.12684187293052673, "metrics/particle/momentum/class_4/event/py_bias": 0.06521191447973251, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8557023406028748, "metrics/particle/momentum/class_4/event/pz_bias": 0.14108917117118835, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.763129949569702, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005440074950456619, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.017998356372117996, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.017396310344338417, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.009168663062155247, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.046227551996707916, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.152272567152977, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.031026596203446388, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6623539924621582, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.030895426869392395, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.027543816715478897, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.026925042271614075, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008065871894359589, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10393379628658295, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5174086689949036, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3974950909614563, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.220623016357422, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.35989686846733093, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7875750064849854, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5189226865768433, "metrics/particle/momentum/class_5/event/px_bias": 0.05583055689930916, "metrics/particle/momentum/class_5/event/py_bias": 0.022404253482818604, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.504798412322998, "metrics/particle/momentum/class_5/event/pz_bias": -0.17814357578754425, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.284268379211426, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6471779346466064, "step_valid_time": 12137.854341506958, "step_total_time": 12139.50152015686, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998501227451873, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05726372916942368, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3031, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2901, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9621890547263682, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9571098647311118, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9596427390009924, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04289013526888816, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.955130320026394, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9576579556731724, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9621890547263682, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0014341647731442, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07007753899049507, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2139, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494007989347537, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9549107142857143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952147785444024, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045089285714285714, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.953125, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9503672379256621, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494007989347537, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043360186378758, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07156361044125648, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1281, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1196, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9359875097580016, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9345284489477786, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06401249024199844, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.930739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9336455893832943, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9321901792673422, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9330739299610895} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_29000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_29000.json new file mode 100644 index 0000000000000000000000000000000000000000..64fe32eea0d906fe4bb3be041a6e10e1e3e70ac0 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_29000.json @@ -0,0 +1 @@ +{"epoch": 1.3738866780367633, "train": {"Classification_binary": 2.387680768966675, "Classification": 0.019083838909864426, "Regression_pt": 0.1131230890750885, "Regression_eta": 0.010672761127352715, "Regression_sin_phi": 0.010030088946223259, "Regression_cos_phi": 0.00817650742828846, "Regression_energy": 0.116155706346035, "Total": 8.13775634765625, "Total_unweighted": 2.664922760799527}, "train_group": {}, "valid": {"Classification_binary": 2.4466072718302407, "Classification": 0.019580262402693432, "Regression_pt": 0.12546759843826294, "Regression_eta": 0.01031800794104735, "Regression_sin_phi": 0.008125058685739836, "Regression_cos_phi": 0.00808307776848475, "Regression_energy": 0.12888175249099731, "Total": 8.446799596150717, "Total_unweighted": 2.7470630295574665, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.020122474059462547, "diagnostic/loss/clic_pf/Classification_binary": 2.4541738033294678, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00757625512778759, "diagnostic/loss/clic_pf/Regression_energy": 0.13307037949562073, "diagnostic/loss/clic_pf/Regression_eta": 0.009681928902864456, "diagnostic/loss/clic_pf/Regression_pt": 0.1296335756778717, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007681354880332947, "diagnostic/regression/clic_pf/energy_pred_mean": 0.018755747005343437, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1833106428384781, "diagnostic/regression/clic_pf/energy_residual_mean": 0.03430936858057976, "diagnostic/regression/clic_pf/energy_residual_rms": 0.414092481136322, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.01254506316035986, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17750701308250427, "diagnostic/regression/clic_pf/pt_residual_mean": 0.05629947781562805, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4164145886898041, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9712763428688049, "metrics/particle/class_1/pid_accuracy": 0.987208902835846, "metrics/particle/class_2/efficiency": 0.4529770016670227, "metrics/particle/class_2/pid_accuracy": 0.7562857866287231, "metrics/particle/class_3/efficiency": 0.7585515975952148, "metrics/particle/class_3/pid_accuracy": 0.987270176410675, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5189743638038635, "metrics/particle/class_5/efficiency": 0.9939637780189514, "metrics/particle/class_5/pid_accuracy": 0.7510121464729309, "metrics/particle/count/bias_mean": -0.7392156720161438, "metrics/particle/count/mae": 3.1143791675567627, "metrics/particle/count/prediction_mean": 69.2601318359375, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0443490631878376, "metrics/particle/event/energy_response_mean": 0.9954763054847717, "metrics/particle/event/met_abs_error": 4.212733745574951, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03405429795384407, "metrics/particle/event/scalar_pt_response_mean": 0.9997483491897583, "metrics/particle/event/vector_pt_closure": 0.036497995257377625, "metrics/particle/matched/delta_eta_abs_mean": 0.008302394300699234, "metrics/particle/matched/delta_phi_abs_mean": 0.00951396208256483, "metrics/particle/matched/delta_r_mean": 0.014180822297930717, "metrics/particle/matched/energy_relative_abs_mean": 0.08253251761198044, "metrics/particle/matched/pid_accuracy": 0.9704716205596924, "metrics/particle/matched/pt_relative_abs_mean": 0.0730217918753624, "metrics/particle/matching/duplicate_fraction": 0.025658689439296722, "metrics/particle/matching/efficiency": 0.8360581994056702, "metrics/particle/matching/f1": 0.8404961824417114, "metrics/particle/matching/purity": 0.8449814915657043, "metrics/particle/matching/target_coverage_dr0p05": 0.8126593232154846, "metrics/particle/matching/target_coverage_dr0p10": 0.8656476736068726, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7475046515464783, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8716234564781189, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01562701351940632, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07900645583868027, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0727306380867958, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010008969344198704, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12838402390480042, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003096330212429166, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31717655062675476, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.005095488857477903, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2241181135177612, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8720.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26248428225517273, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8365607261657715, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8257843255996704, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7104616761207581, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11721295863389969, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.140512466430664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01313015166670084, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6930240392684937, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.034153081476688385, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5635430812835693, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10434.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8112382292747498, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.6389731764793396, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.852398872375488, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.541550874710083, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2176394760608673, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.3353677988052368, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.0592241287231445, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3294996917247772, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.631133079528809, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02570158988237381, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08859271556138992, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08736294507980347, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02096557803452015, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14280690252780914, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.015889830887317657, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8654285073280334, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.014412941411137581, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8249268531799316, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1888.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7040481567382812, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.245888113975525, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.6428275108337402, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.487837791442871, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23660872876644135, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 23.48615074157715, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06881873309612274, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.612962245941162, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.05410772189497948, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.040315628051758, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2819.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6145833134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.42238187789917, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.08420991897583, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.02162504196167, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12560252845287323, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11339568346738815, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.948215961456299, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.391025185585022, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.090392112731934, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012552453204989433, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08407489210367203, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07252433896064758, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010398773476481438, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12873145937919617, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006740990211255848, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10530415177345276, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005195274483412504, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5858533382415771, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19285.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1343616396188736, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5717723965644836, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4313502907752991, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3311404883861542, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07980675250291824, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.149158954620361, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002369117923080921, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21384868025779724, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004818515852093697, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.0562424659729004, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21527.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8784276247024536, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.9163419008255005, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.535515785217285, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.547335624694824, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.6303296089172363, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.05305365100502968, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.6986162662506104, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.10047994554042816, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.915869235992432, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014279805123806, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08288172632455826, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0735217034816742, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010952448472380638, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12956513464450836, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002341685350984335, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2151172161102295, "metrics/particle/momentum/all/accepted/pz_bias": -0.0009549583191983402, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0782195329666138, "metrics/particle/momentum/all/accepted/samples": 29893.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.218972846865654, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7058475017547607, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.728925883769989, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6197421550750732, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10373777896165848, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.999970436096191, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01098332367837429, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5520550012588501, "metrics/particle/momentum/all/assigned/pz_bias": 0.011649073101580143, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8110170364379883, "metrics/particle/momentum/all/assigned/samples": 34780.0, "metrics/particle/momentum/all/efficiency": 0.835583508014679, "metrics/particle/momentum/all/event/energy_bias": -3.9776968955993652, "metrics/particle/momentum/all/event/p3_error_mean": 12.017951011657715, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.803731918334961, "metrics/particle/momentum/all/event/px_bias": 0.7223665714263916, "metrics/particle/momentum/all/event/py_bias": 0.5018171668052673, "metrics/particle/momentum/all/event/pz_abs_mean": 9.174474716186523, "metrics/particle/momentum/all/event/pz_bias": -0.16200542449951172, "metrics/particle/momentum/all/event/pz_residual_rms": 16.315914154052734, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006674930453300476, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03869131952524185, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021455425769090652, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005663353484123945, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06195239722728729, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013449076795950532, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11126953363418579, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.007075952365994453, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.844452977180481, "metrics/particle/momentum/class_1/accepted/samples": 16358.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.039652105420827866, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10505814105272293, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08929195255041122, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06234584376215935, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.01944279856979847, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7644919157028198, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0045643155463039875, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.22328433394432068, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02077234722673893, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.080998420715332, "metrics/particle/momentum/class_1/assigned/samples": 16870.0, "metrics/particle/momentum/class_1/efficiency": 0.9694778919219971, "metrics/particle/momentum/class_1/event/energy_bias": 1.4496731758117676, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.251836776733398, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.208124160766602, "metrics/particle/momentum/class_1/event/px_bias": 0.12584330141544342, "metrics/particle/momentum/class_1/event/py_bias": -0.4913176894187927, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.2250518798828125, "metrics/particle/momentum/class_1/event/pz_bias": 0.2365642488002777, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.500608444213867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037001047283411026, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19742120802402496, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1979982703924179, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.048473063856363297, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2393207997083664, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02996254712343216, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6319561004638672, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.002668430795893073, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6159772872924805, "metrics/particle/momentum/class_2/accepted/samples": 1335.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8621776103973389, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6198332905769348, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7286788821220398, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.229971781373024, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3444612920284271, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.1110281944274902, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07207890599966049, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.928250551223755, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05834188312292099, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.942124366760254, "metrics/particle/momentum/class_2/assigned/samples": 2636.0, "metrics/particle/momentum/class_2/efficiency": 0.45315682888031006, "metrics/particle/momentum/class_2/event/energy_bias": -8.996715545654297, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.283181190490723, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.418848037719727, "metrics/particle/momentum/class_2/event/px_bias": 0.5619348883628845, "metrics/particle/momentum/class_2/event/py_bias": -0.17747023701667786, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.003369331359863, "metrics/particle/momentum/class_2/event/pz_bias": -0.1251862645149231, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.696375846862793, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.022561829537153244, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.133636936545372, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1337442249059677, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02534719929099083, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17533601820468903, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0006849901401437819, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1996781826019287, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.010976352728903294, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6781443357467651, "metrics/particle/momentum/class_3/accepted/samples": 11679.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3146706223487854, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4209918975830078, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4777449369430542, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3414980173110962, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15987780690193176, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 13.625133514404297, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0076714190654456615, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5111076235771179, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.0010085615795105696, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.199554681777954, "metrics/particle/momentum/class_3/assigned/samples": 14730.0, "metrics/particle/momentum/class_3/efficiency": 0.7577861547470093, "metrics/particle/momentum/class_3/event/energy_bias": 4.600716590881348, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.701723098754883, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.708740234375, "metrics/particle/momentum/class_3/event/px_bias": -0.11080292612314224, "metrics/particle/momentum/class_3/event/py_bias": 1.0255649089813232, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.362215995788574, "metrics/particle/momentum/class_3/event/pz_bias": -0.10828053951263428, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.516324996948242, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011619152501225471, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05225455388426781, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05274335294961929, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006963982246816158, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09791578352451324, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.233193039894104, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.010852172039449215, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8063645362854004, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.0834014043211937, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5732540488243103, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.42311063408851624, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3734453022480011, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.558316707611084, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2771388292312622, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.026013677939772606, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9081183671951294, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -0.7089976668357849, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2249985933303833, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8133002519607544, "metrics/particle/momentum/class_4/event/px_bias": 0.10217603296041489, "metrics/particle/momentum/class_4/event/py_bias": 0.11523102968931198, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7433380484580994, "metrics/particle/momentum/class_4/event/pz_bias": 0.044622331857681274, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.621079921722412, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00451365951448679, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.013947786763310432, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.013696680776774883, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005923732183873653, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.037320490926504135, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11115826666355133, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.049320563673973083, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4643824100494385, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04920433461666107, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.02437712624669075, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02493867836892605, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.006591372657567263, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10050012171268463, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.47175732254981995, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41093894839286804, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.172719478607178, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.3223734498023987, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7560459971427917, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.4794894754886627, "metrics/particle/momentum/class_5/event/px_bias": 0.04321521520614624, "metrics/particle/momentum/class_5/event/py_bias": 0.029809093102812767, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5051230788230896, "metrics/particle/momentum/class_5/event/pz_bias": -0.20972520112991333, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.292149305343628, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5887134075164795, "step_valid_time": 12572.331155776978, "step_total_time": 12573.919869422913, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991974496070979, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058772041341145775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2891, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9608623548922056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9592715231788079, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.96006628003314, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04072847682119205, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9588723051409619, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9572847682119205, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9580778790389395, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9608623548922056, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9998860527228666, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06474233742682434, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2247, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2139, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507323568575233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9532710280373832, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.952, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04672897196261682, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9494007989347537, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.951935914552737, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9506666666666667, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507323568575233, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992374258349489, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06613914886518657, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1277, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.932295719844358, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9381362568519969, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9352068696330992, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06186374314800313, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9357870007830854, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9328649492583919, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.932295719844358} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_3000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_3000.json new file mode 100644 index 0000000000000000000000000000000000000000..a05aa519a750532ddaa2b4cbae643c39f116b739 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_3000.json @@ -0,0 +1 @@ +{"epoch": 0.14212620807276863, "train": {"Classification_binary": 3.4849092960357666, "Classification": 0.03162151575088501, "Regression_pt": 0.17534767091274261, "Regression_eta": 0.012108966708183289, "Regression_sin_phi": 0.009779256768524647, "Regression_cos_phi": 0.010658932849764824, "Regression_energy": 0.17685019969940186, "Total": 11.757617950439453, "Total_unweighted": 3.901275838725269}, "train_group": {}, "valid": {"Classification_binary": 3.3714319864908853, "Classification": 0.030619800090789795, "Regression_pt": 0.18091466029485068, "Regression_eta": 0.01301265632112821, "Regression_sin_phi": 0.009893826519449552, "Regression_cos_phi": 0.00992785394191742, "Regression_energy": 0.18420527378718057, "Total": 11.789788564046225, "Total_unweighted": 3.8000060574462013, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.03139146789908409, "diagnostic/loss/clic_pf/Classification_binary": 3.3541343212127686, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009692744351923466, "diagnostic/loss/clic_pf/Regression_energy": 0.18655063211917877, "diagnostic/loss/clic_pf/Regression_eta": 0.012169740162789822, "diagnostic/loss/clic_pf/Regression_pt": 0.18432289361953735, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009738342836499214, "diagnostic/regression/clic_pf/energy_pred_mean": 0.058564670383930206, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2246341109275818, "diagnostic/regression/clic_pf/energy_residual_mean": 0.07411829382181168, "diagnostic/regression/clic_pf/energy_residual_rms": 0.5112009048461914, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.02784046344459057, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.2221834808588028, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0966849997639656, "diagnostic/regression/clic_pf/pt_residual_rms": 0.5171916484832764, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9524519443511963, "metrics/particle/class_1/pid_accuracy": 0.9864969253540039, "metrics/particle/class_2/efficiency": 0.32566532492637634, "metrics/particle/class_2/pid_accuracy": 0.5193905830383301, "metrics/particle/class_3/efficiency": 0.6894277334213257, "metrics/particle/class_3/pid_accuracy": 0.9777089953422546, "metrics/particle/class_4/efficiency": 0.932170569896698, "metrics/particle/class_4/pid_accuracy": 0.1392931342124939, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.44989773631095886, "metrics/particle/count/bias_mean": -1.0267974138259888, "metrics/particle/count/mae": 4.032026290893555, "metrics/particle/count/prediction_mean": 68.97254943847656, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.08425693213939667, "metrics/particle/event/energy_response_mean": 1.045846939086914, "metrics/particle/event/met_abs_error": 8.881622314453125, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07485288381576538, "metrics/particle/event/scalar_pt_response_mean": 1.049278736114502, "metrics/particle/event/vector_pt_closure": 0.0694759413599968, "metrics/particle/matched/delta_eta_abs_mean": 0.01785409264266491, "metrics/particle/matched/delta_phi_abs_mean": 0.01430454384535551, "metrics/particle/matched/delta_r_mean": 0.025309138000011444, "metrics/particle/matched/energy_relative_abs_mean": 0.08822821825742722, "metrics/particle/matched/pid_accuracy": 0.9543566107749939, "metrics/particle/matched/pt_relative_abs_mean": 0.08046647161245346, "metrics/particle/matching/duplicate_fraction": 0.03979986533522606, "metrics/particle/matching/efficiency": 0.7865619659423828, "metrics/particle/matching/f1": 0.7923734784126282, "metrics/particle/matching/purity": 0.7982715368270874, "metrics/particle/matching/target_coverage_dr0p05": 0.7230599522590637, "metrics/particle/matching/target_coverage_dr0p10": 0.8225940465927124, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6967291831970215, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8303812146186829, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02771700546145439, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08629534393548965, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07932672649621964, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.0263838954269886, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13427554070949554, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004108054097741842, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.36448991298675537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.008345372043550014, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.265751600265503, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8033.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.36338600516319275, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.5809118747711182, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.8696885108947754, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.7562310695648193, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.15851880609989166, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 20.55436134338379, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02118767611682415, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 1.0150679349899292, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.017334992066025734, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.492889642715454, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10289.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7473253607749939, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 2.939004898071289, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.511938095092773, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.457462310791016, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.4264131784439087, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.03987102955579758, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 9.127889633178711, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.29607099294662476, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.210866928100586, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03372042626142502, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.1007506400346756, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09407638013362885, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.03379098325967789, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15364179015159607, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018999366089701653, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9501403570175171, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.007355029694736004, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8533122539520264, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1579.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8361093401908875, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.8671737909317017, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.7143564224243164, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.56584095954895, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.3029974699020386, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 24.251136779785156, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.09209100902080536, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.9802494049072266, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.031445350497961044, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 14.890695571899414, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2769.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5139973759651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 2.103679656982422, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.982423782348633, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.7640882730484009, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.10754390060901642, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08539554476737976, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 9.723626136779785, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.502438485622406, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 21.574663162231445, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.023786628618836403, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08809659630060196, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0802030935883522, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01717688888311386, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13450494408607483, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007558171055279672, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.13509061932563782, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.037227813154459, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6468695998191833, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18523.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.18715709447860718, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1663504838943481, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9089539051055908, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8046647906303406, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.10516975820064545, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.547831535339355, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.004763461649417877, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.3935759961605072, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.06391776353120804, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 5.119900703430176, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21413.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8437187075614929, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.098138332366943, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 13.087542533874512, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.590596199035645, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.574598550796509, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.5055055618286133, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 6.768682479858398, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 2.5990724563598633, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 32.272613525390625, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.025466322898864746, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08829247951507568, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.08073148131370544, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.020738059654831886, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13558557629585266, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002736804774031043, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.24633029103279114, "metrics/particle/momentum/all/accepted/pz_bias": 0.026479320600628853, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0907940864562988, "metrics/particle/momentum/all/accepted/samples": 28135.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.29188767075538635, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.3463859558105469, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.4210700988769531, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.3104918003082275, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1369847059249878, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 15.16490364074707, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01668068766593933, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.8671923279762268, "metrics/particle/momentum/all/assigned/pz_bias": 0.04740515351295471, "metrics/particle/momentum/all/assigned/pz_residual_rms": 6.143002033233643, "metrics/particle/momentum/all/assigned/samples": 34471.0, "metrics/particle/momentum/all/efficiency": 0.7864430546760559, "metrics/particle/momentum/all/event/energy_bias": 11.14082145690918, "metrics/particle/momentum/all/event/p3_error_mean": 23.997713088989258, "metrics/particle/momentum/all/event/pt_vector_error_mean": 12.54823112487793, "metrics/particle/momentum/all/event/px_bias": -2.2557291984558105, "metrics/particle/momentum/all/event/py_bias": -1.459981083869934, "metrics/particle/momentum/all/event/pz_abs_mean": 17.864356994628906, "metrics/particle/momentum/all/event/pz_bias": 3.3975818157196045, "metrics/particle/momentum/all/event/pz_residual_rms": 39.7297477722168, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.017979182302951813, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04107574746012688, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02961084432899952, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.016267796978354454, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0660543441772461, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010594540508463979, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13646060228347778, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.049416761845350266, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7445358633995056, "metrics/particle/momentum/class_1/accepted/samples": 16046.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07282447814941406, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.20011818408966064, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1867256909608841, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.15798169374465942, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.03259645402431488, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.4763166904449463, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005452497862279415, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3617055118083954, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.09168191999197006, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 5.800536155700684, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9509868025779724, "metrics/particle/momentum/class_1/event/energy_bias": 9.120744705200195, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.586679458618164, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.314018249511719, "metrics/particle/momentum/class_1/event/px_bias": -1.9894585609436035, "metrics/particle/momentum/class_1/event/py_bias": -0.7094598412513733, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.43405818939209, "metrics/particle/momentum/class_1/event/pz_bias": 3.2745447158813477, "metrics/particle/momentum/class_1/event/pz_residual_rms": 35.153438568115234, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.045541439205408096, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21404187381267548, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21179574728012085, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.06287344545125961, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2495443969964981, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04865424335002899, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 2.013014554977417, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.007766611874103546, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.332178592681885, "metrics/particle/momentum/class_2/accepted/samples": 966.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0835158824920654, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 1.0386160612106323, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.3357410430908203, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.8067928552627563, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.44026899337768555, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 5.95810079574585, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09889240562915802, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 4.018945693969727, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.16579319536685944, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.919621467590332, "metrics/particle/momentum/class_2/assigned/samples": 2528.0, "metrics/particle/momentum/class_2/efficiency": 0.327902227640152, "metrics/particle/momentum/class_2/event/energy_bias": -4.946822166442871, "metrics/particle/momentum/class_2/event/p3_error_mean": 21.558393478393555, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 10.834351539611816, "metrics/particle/momentum/class_2/event/px_bias": -1.0709617137908936, "metrics/particle/momentum/class_2/event/py_bias": -0.900478720664978, "metrics/particle/momentum/class_2/event/pz_abs_mean": 16.08074378967285, "metrics/particle/momentum/class_2/event/pz_bias": -0.012710630893707275, "metrics/particle/momentum/class_2/event/pz_residual_rms": 26.394983291625977, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03528910502791405, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1506125032901764, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1481139361858368, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.035941239446401596, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19011341035366058, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011314350413158536, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.24783997237682343, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01245885994285345, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7420668601989746, "metrics/particle/momentum/class_3/accepted/samples": 10606.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.41694125533103943, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.735825538635254, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.8952956199645996, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.7599682807922363, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.20914222300052643, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.039146423339844, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01590251922607422, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.9197623133659363, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03418607637286186, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 5.751964569091797, "metrics/particle/momentum/class_3/assigned/samples": 14526.0, "metrics/particle/momentum/class_3/efficiency": 0.6881650686264038, "metrics/particle/momentum/class_3/event/energy_bias": 10.776081085205078, "metrics/particle/momentum/class_3/event/p3_error_mean": 14.084541320800781, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 7.071989059448242, "metrics/particle/momentum/class_3/event/px_bias": 0.21723473072052002, "metrics/particle/momentum/class_3/event/py_bias": -0.0670168548822403, "metrics/particle/momentum/class_3/event/pz_abs_mean": 10.721358299255371, "metrics/particle/momentum/class_3/event/pz_bias": 0.11195138096809387, "metrics/particle/momentum/class_3/event/pz_residual_rms": 19.477033615112305, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.02042868174612522, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.047456663101911545, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.047499075531959534, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0013789694057777524, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09306642413139343, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.31194254755973816, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.1523483395576477, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.1281375885009766, "metrics/particle/momentum/class_4/accepted/samples": 339.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09107118844985962, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8252662420272827, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.0875648260116577, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.0385801792144775, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.04696132615208626, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 8.108269691467285, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.40642204880714417, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.07885029911994934, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3595709800720215, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9364640712738037, "metrics/particle/momentum/class_4/event/energy_bias": -2.537583827972412, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.792724609375, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.9382765293121338, "metrics/particle/momentum/class_4/event/px_bias": 0.41785985231399536, "metrics/particle/momentum/class_4/event/py_bias": 0.07354758679866791, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6549570560455322, "metrics/particle/momentum/class_4/event/pz_bias": 0.34992292523384094, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.901556491851807, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.01576658897101879, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026736589148640633, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026130633428692818, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00847020372748375, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06626294553279877, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.00561797758564353, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3480042517185211, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.22499655187129974, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 1.004565954208374, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.023673031479120255, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.042329221963882446, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.04160894826054573, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00824225414544344, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1292387992143631, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6727645993232727, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.10357900708913803, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.117547988891602, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -1.2715975046157837, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.8214972019195557, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.241553544998169, "metrics/particle/momentum/class_5/event/px_bias": 0.16959628462791443, "metrics/particle/momentum/class_5/event/py_bias": 0.143426775932312, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0775856971740723, "metrics/particle/momentum/class_5/event/pz_bias": -0.3261266350746155, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.629749774932861, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.522768497467041, "step_valid_time": 1306.9021453857422, "step_total_time": 1308.4249141216278, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0156201108024938, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1235924939530374, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3006, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2801, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9290215588723051, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9318030605455755, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9304102308586614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06819693945442448, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203980099502488, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9231536926147704, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9217737917289487, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9290215588723051, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0175539756688128, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.13772255151259405, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2062, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2038, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9152241455836663, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9238351254480287, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9195094760312151, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07616487455197132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9045716822015091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9130824372759857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9088071348940914, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9152241455836663, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0332054942128688, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14368648571596243, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1128, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1114, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8778210116731517, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8798751950078003, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8788469029996104, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12012480499219969, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8669260700389105, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8689547581903276, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8679392286716011, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8778210116731517} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_30000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..4fed5a44ea96950781453c294eb50ae749f731cf --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 1.4212620807276861, "train": {"Classification_binary": 2.39528226852417, "Classification": 0.02013508602976799, "Regression_pt": 0.11616106331348419, "Regression_eta": 0.00853718165308237, "Regression_sin_phi": 0.007509915623813868, "Regression_cos_phi": 0.00719718961045146, "Regression_energy": 0.12089590728282928, "Total": 7.986556053161621, "Total_unweighted": 2.675718612037599}, "train_group": {}, "valid": {"Classification_binary": 2.417469342549642, "Classification": 0.019335446258385975, "Regression_pt": 0.12384925285975139, "Regression_eta": 0.010136800507704416, "Regression_sin_phi": 0.007959092035889626, "Regression_cos_phi": 0.00802114667991797, "Regression_energy": 0.12715710202852884, "Total": 8.33670171101888, "Total_unweighted": 2.7139281829198203, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019782884046435356, "diagnostic/loss/clic_pf/Classification_binary": 2.4282240867614746, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007543244399130344, "diagnostic/loss/clic_pf/Regression_energy": 0.13012154400348663, "diagnostic/loss/clic_pf/Regression_eta": 0.009593702852725983, "diagnostic/loss/clic_pf/Regression_pt": 0.12674972414970398, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007512093521654606, "diagnostic/regression/clic_pf/energy_pred_mean": 0.035137444734573364, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18694046139717102, "diagnostic/regression/clic_pf/energy_residual_mean": 0.050691068172454834, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42092397809028625, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.002190131228417158, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17861102521419525, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06665440648794174, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42088520526885986, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.971713662147522, "metrics/particle/class_1/pid_accuracy": 0.9874601364135742, "metrics/particle/class_2/efficiency": 0.44260260462760925, "metrics/particle/class_2/pid_accuracy": 0.7704458832740784, "metrics/particle/class_3/efficiency": 0.7546718120574951, "metrics/particle/class_3/pid_accuracy": 0.9860337376594543, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5169230699539185, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7484787106513977, "metrics/particle/count/bias_mean": -0.6346405148506165, "metrics/particle/count/mae": 3.063398599624634, "metrics/particle/count/prediction_mean": 69.36470794677734, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04804336279630661, "metrics/particle/event/energy_response_mean": 1.017218828201294, "metrics/particle/event/met_abs_error": 4.3962249755859375, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03604147210717201, "metrics/particle/event/scalar_pt_response_mean": 1.0102901458740234, "metrics/particle/event/vector_pt_closure": 0.038382094353437424, "metrics/particle/matched/delta_eta_abs_mean": 0.008477789349853992, "metrics/particle/matched/delta_phi_abs_mean": 0.009255558252334595, "metrics/particle/matched/delta_r_mean": 0.014109908603131771, "metrics/particle/matched/energy_relative_abs_mean": 0.08378084748983383, "metrics/particle/matched/pid_accuracy": 0.9709038138389587, "metrics/particle/matched/pt_relative_abs_mean": 0.0725323036313057, "metrics/particle/matching/duplicate_fraction": 0.02623247355222702, "metrics/particle/matching/efficiency": 0.8337146043777466, "metrics/particle/matching/f1": 0.8375111818313599, "metrics/particle/matching/purity": 0.8413425087928772, "metrics/particle/matching/target_coverage_dr0p05": 0.815609872341156, "metrics/particle/matching/target_coverage_dr0p10": 0.866077184677124, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7448809146881104, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8700921535491943, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01693849079310894, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07946978509426117, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07140633463859558, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012295321561396122, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12666602432727814, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0029837044421583414, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3093257248401642, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0013927503023296595, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.188049554824829, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8714.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26965776085853577, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.88135826587677, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.906005859375, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7951691150665283, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11596284806728363, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.630147933959961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01254428829997778, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7011731863021851, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.01019472349435091, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.921966791152954, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10443.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.810680091381073, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.8248848915100098, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.90319299697876, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5935635566711426, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2886859178543091, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.15219047665596008, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.105139255523682, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.045368097722530365, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.741785049438477, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.025663867592811584, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09426965564489365, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08778245002031326, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02301868423819542, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1450459510087967, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02076677232980728, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.925227701663971, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0974850058555603, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.945768117904663, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1878.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7077425122261047, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2292777299880981, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.1117045879364014, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9723329544067383, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24457663297653198, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.632467269897461, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06892932206392288, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5453579425811768, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.06971819698810577, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.986077785491943, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2858.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.611328125, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.5444421768188477, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.972815036773682, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0464762449264526, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.034169845283031464, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.021675340831279755, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.808290481567383, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4272961914539337, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 12.735161781311035, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011844686232507229, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0856962502002716, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07269547134637833, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01613779179751873, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12976014614105225, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00046779977856203914, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10296940803527832, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.005046875216066837, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5502888560295105, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19239.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13238997757434845, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5775529742240906, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.43697389960289, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.34245240688323975, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08231140673160553, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.514716625213623, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029264213517308235, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22930096089839935, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.004933787044137716, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2212480306625366, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21528.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8763323426246643, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.33947303891181946, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.782450199127197, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.762167930603027, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.24252551794052124, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.07362140715122223, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.8099234104156494, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.5600267648696899, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.370188236236572, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014202630147337914, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08441715687513351, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07326868921518326, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015448542311787605, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12988603115081787, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002480641007423401, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21501359343528748, "metrics/particle/momentum/all/accepted/pz_bias": 0.0024753969628363848, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0741947889328003, "metrics/particle/momentum/all/accepted/samples": 29831.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22076013684272766, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7221241593360901, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7150318622589111, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6119381785392761, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10571650415658951, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.574583053588867, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011226276867091656, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5608364939689636, "metrics/particle/momentum/all/assigned/pz_bias": 0.005728086456656456, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.736185073852539, "metrics/particle/momentum/all/assigned/samples": 34829.0, "metrics/particle/momentum/all/efficiency": 0.8338504433631897, "metrics/particle/momentum/all/event/energy_bias": 3.0298542976379395, "metrics/particle/momentum/all/event/p3_error_mean": 12.360910415649414, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.028097629547119, "metrics/particle/momentum/all/event/px_bias": 0.5653812885284424, "metrics/particle/momentum/all/event/py_bias": 0.24748721718788147, "metrics/particle/momentum/all/event/pz_abs_mean": 9.363131523132324, "metrics/particle/momentum/all/event/pz_bias": -0.08736243844032288, "metrics/particle/momentum/all/event/pz_residual_rms": 14.902881622314453, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007204721216112375, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03873514384031296, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02003200352191925, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0025316423270851374, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.059564221650362015, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010989682050421834, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10329295694828033, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.002060903003439307, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7956075668334961, "metrics/particle/momentum/class_1/accepted/samples": 16379.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03856108710169792, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1116357073187828, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08367857336997986, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05627233535051346, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017840208485722542, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7330313920974731, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089615773409605, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.20969118177890778, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.013523848727345467, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6889485120773315, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9707224369049072, "metrics/particle/momentum/class_1/event/energy_bias": 3.010118246078491, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.546774864196777, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.3177690505981445, "metrics/particle/momentum/class_1/event/px_bias": -0.22961099445819855, "metrics/particle/momentum/class_1/event/py_bias": -0.3283684253692627, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.525662422180176, "metrics/particle/momentum/class_1/event/pz_bias": -0.4951309263706207, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.231610298156738, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03651665151119232, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19352416694164276, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19425533711910248, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.025281593203544617, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23428001999855042, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03292496129870415, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6450695991516113, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.0569453164935112, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6520748138427734, "metrics/particle/momentum/class_2/accepted/samples": 1306.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8810281157493591, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7423542141914368, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8620709180831909, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3890010118484497, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.349507212638855, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.524596691131592, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07429870963096619, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0451908111572266, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.0074035003781318665, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.092843532562256, "metrics/particle/momentum/class_2/assigned/samples": 2638.0, "metrics/particle/momentum/class_2/efficiency": 0.44331297278404236, "metrics/particle/momentum/class_2/event/energy_bias": -5.812033653259277, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.62454605102539, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.444648265838623, "metrics/particle/momentum/class_2/event/px_bias": 0.5105587244033813, "metrics/particle/momentum/class_2/event/py_bias": -0.1903119832277298, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.357943534851074, "metrics/particle/momentum/class_2/event/pz_bias": 0.34790900349617004, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.053130149841309, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021783141419291496, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1386583298444748, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13632620871067047, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03879369795322418, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17846055328845978, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011183758033439517, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21334803104400635, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.002660132944583893, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7420104146003723, "metrics/particle/momentum/class_3/accepted/samples": 11624.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3165246844291687, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4275885820388794, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4251270294189453, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2994043827056885, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16548223793506622, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.87894344329834, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008392554707825184, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5258191823959351, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.002852070378139615, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.270551919937134, "metrics/particle/momentum/class_3/assigned/samples": 14775.0, "metrics/particle/momentum/class_3/efficiency": 0.7542175054550171, "metrics/particle/momentum/class_3/event/energy_bias": 6.383937358856201, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.959555625915527, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.805468559265137, "metrics/particle/momentum/class_3/event/px_bias": -0.05987803637981415, "metrics/particle/momentum/class_3/event/py_bias": 0.6767513155937195, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.55910587310791, "metrics/particle/momentum/class_3/event/pz_bias": 0.07390524446964264, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.337434768676758, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011349206790328026, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.04802694916725159, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04833332821726799, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0004543580871541053, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09207780659198761, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.20706261694431305, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.005165113136172295, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7558925747871399, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06696872413158417, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5846329927444458, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4335780739784241, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3824865520000458, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.033149171620607376, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.8556082248687744, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2608901262283325, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.038573089987039566, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8661351203918457, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.4017907679080963, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.494537115097046, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9621345400810242, "metrics/particle/momentum/class_4/event/px_bias": 0.2422785460948944, "metrics/particle/momentum/class_4/event/py_bias": 0.04658469930291176, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9538887143135071, "metrics/particle/momentum/class_4/event/pz_bias": 0.15099723637104034, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.8530421257019043, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004926132969558239, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015783162787556648, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01476217620074749, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007283254526555538, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.044565360993146896, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12734052538871765, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.008880000561475754, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4678272604942322, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.07255706191062927, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.02613222226500511, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02565823867917061, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.002318908227607608, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.10123670846223831, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5430302023887634, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.20503462851047516, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3664870262146, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -0.15037661790847778, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9655153751373291, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6222689747810364, "metrics/particle/momentum/class_5/event/px_bias": 0.10203303396701813, "metrics/particle/momentum/class_5/event/py_bias": 0.042831603437662125, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.6199132204055786, "metrics/particle/momentum/class_5/event/pz_bias": -0.16504311561584473, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.44610333442688, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.691208839416504, "step_valid_time": 13005.95936870575, "step_total_time": 13007.650577783585, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988601131158039, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05738762907623707, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2880, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552238805970149, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9530112508272667, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9541162829219811, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04698874917273329, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9538971807628525, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9516876240900066, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9527911214179228, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552238805970149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018576611154866, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06927186751246106, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2138, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9507323568575233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9536954585930543, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9522116025783507, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04630454140694568, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9489569462938304, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519145146927872, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9504334296510336, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9507323568575233, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004111259498955, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07134810072143205, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1283, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1193, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9284046692607004, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9298519095869057, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9291277258566978, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0701480904130943, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9268482490272374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9282930631332814, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.927570093457944, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9284046692607004} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_31000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_31000.json new file mode 100644 index 0000000000000000000000000000000000000000..90a2fad20a736e9d73172d574c28842d4aef52d4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_31000.json @@ -0,0 +1 @@ +{"epoch": 1.4686374834186091, "train": {"Classification_binary": 2.3765289783477783, "Classification": 0.01851653680205345, "Regression_pt": 0.11168485879898071, "Regression_eta": 0.008430656045675278, "Regression_sin_phi": 0.009409562684595585, "Regression_cos_phi": 0.009537593461573124, "Regression_energy": 0.11173282563686371, "Total": 7.9652886390686035, "Total_unweighted": 2.64584101177752}, "train_group": {}, "valid": {"Classification_binary": 2.4071613947550454, "Classification": 0.019188946733872097, "Regression_pt": 0.1237049400806427, "Regression_eta": 0.010149684424201647, "Regression_sin_phi": 0.008010769883791605, "Regression_cos_phi": 0.007972159733374914, "Regression_energy": 0.12711195151011148, "Total": 8.317773818969727, "Total_unweighted": 2.70329984712104, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01959068514406681, "diagnostic/loss/clic_pf/Classification_binary": 2.4118499755859375, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007672353647649288, "diagnostic/loss/clic_pf/Regression_energy": 0.12928850948810577, "diagnostic/loss/clic_pf/Regression_eta": 0.00935257039964199, "diagnostic/loss/clic_pf/Regression_pt": 0.12606389820575714, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007638539187610149, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02767343446612358, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1838023066520691, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04322705790400505, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4167446494102478, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.006565301213413477, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17725874483585358, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06227923929691315, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41722068190574646, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9697059988975525, "metrics/particle/class_1/pid_accuracy": 0.9874136447906494, "metrics/particle/class_2/efficiency": 0.4421515464782715, "metrics/particle/class_2/pid_accuracy": 0.7679163217544556, "metrics/particle/class_3/efficiency": 0.7574307322502136, "metrics/particle/class_3/pid_accuracy": 0.9871659874916077, "metrics/particle/class_4/efficiency": 0.940891444683075, "metrics/particle/class_4/pid_accuracy": 0.5355303883552551, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7581300735473633, "metrics/particle/count/bias_mean": -1.314379096031189, "metrics/particle/count/mae": 3.211111068725586, "metrics/particle/count/prediction_mean": 68.68496704101562, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.045734960585832596, "metrics/particle/event/energy_response_mean": 1.0001856088638306, "metrics/particle/event/met_abs_error": 4.239430904388428, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03490127995610237, "metrics/particle/event/scalar_pt_response_mean": 1.0007734298706055, "metrics/particle/event/vector_pt_closure": 0.037023067474365234, "metrics/particle/matched/delta_eta_abs_mean": 0.007883197627961636, "metrics/particle/matched/delta_phi_abs_mean": 0.010305898264050484, "metrics/particle/matched/delta_r_mean": 0.014450933784246445, "metrics/particle/matched/energy_relative_abs_mean": 0.08195726573467255, "metrics/particle/matched/pid_accuracy": 0.9715031385421753, "metrics/particle/matched/pt_relative_abs_mean": 0.07277258485555649, "metrics/particle/matching/duplicate_fraction": 0.02419876679778099, "metrics/particle/matching/efficiency": 0.8338826894760132, "metrics/particle/matching/f1": 0.8417857885360718, "metrics/particle/matching/purity": 0.8498401045799255, "metrics/particle/matching/target_coverage_dr0p05": 0.8140318989753723, "metrics/particle/matching/target_coverage_dr0p10": 0.8641443848609924, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7461694478988647, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8689156770706177, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.016393225640058517, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07795105129480362, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0720212310552597, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012546674348413944, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12680572271347046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0033425542060285807, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3134680986404419, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.01806253008544445, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.235006332397461, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8676.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26107949018478394, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.900632381439209, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8894860744476318, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7780922055244446, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11276596039533615, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.886289596557617, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013056092895567417, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6920880079269409, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.0028935407754033804, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6172935962677, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10340.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8071448802947998, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.5869069695472717, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.952434539794922, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.5591652393341064, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.00021587777882814407, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.1002596989274025, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 5.203381061553955, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.4293067455291748, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.842883110046387, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.024239273741841316, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0879940614104271, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08484389632940292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01785028725862503, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13880570232868195, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.017506631091237068, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8241721391677856, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09666924178600311, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.623626470565796, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1885.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6762735843658447, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0902369022369385, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.9389804601669312, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.7904253005981445, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23418621718883514, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 12.494803428649902, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06609807908535004, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4651618003845215, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09297244995832443, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.641993522644043, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2814.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6136067509651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.0487903356552124, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.482600212097168, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0280543565750122, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.07576016336679459, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.07364042848348618, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.3466997146606445, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.23711858689785004, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.089725494384766, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012833135202527046, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08348345011472702, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07240638136863708, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014287691563367844, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12803363800048828, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000467411067802459, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10210712254047394, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.021894963458180428, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5411933660507202, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19255.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13281069695949554, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.540364146232605, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.39543548226356506, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.2990225553512573, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07985463738441467, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 3.6739184856414795, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024692509323358536, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2197076380252838, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.027520641684532166, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2957303524017334, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21464.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.877061128616333, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.6181177496910095, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.633077144622803, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.562155723571777, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.027605220675468445, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.580329179763794, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.8485560417175293, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.47869959473609924, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.690876483917236, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014590175822377205, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08215877413749695, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07308061420917511, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014006313867866993, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1283872276544571, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002381271915510297, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20925965905189514, "metrics/particle/momentum/all/accepted/pz_bias": -0.013284025713801384, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0335112810134888, "metrics/particle/momentum/all/accepted/samples": 29816.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21529969573020935, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6926697492599487, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6684731245040894, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5633471608161926, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10223005712032318, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 6.681746482849121, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010803627781569958, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5433289408683777, "metrics/particle/momentum/all/assigned/pz_bias": -0.008641728200018406, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7989695072174072, "metrics/particle/momentum/all/assigned/samples": 34618.0, "metrics/particle/momentum/all/efficiency": 0.8334311842918396, "metrics/particle/momentum/all/event/energy_bias": -2.253815174102783, "metrics/particle/momentum/all/event/p3_error_mean": 12.451617240905762, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.766626834869385, "metrics/particle/momentum/all/event/px_bias": -0.0483708381652832, "metrics/particle/momentum/all/event/py_bias": -0.6069485545158386, "metrics/particle/momentum/all/event/pz_abs_mean": 9.726258277893066, "metrics/particle/momentum/all/event/pz_bias": -1.67088782787323, "metrics/particle/momentum/all/event/pz_residual_rms": 17.486757278442383, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006874409504234791, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03763760253787041, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.021334432065486908, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.007977518253028393, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05959320068359375, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011013215407729149, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10447829216718674, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0036445974837988615, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7845653295516968, "metrics/particle/momentum/class_1/accepted/samples": 16344.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.041658610105514526, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1010415330529213, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08569081872701645, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.060914162546396255, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.020210999995470047, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6640090942382812, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0036154575645923615, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2206106036901474, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.017085177823901176, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.253216505050659, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9686481356620789, "metrics/particle/momentum/class_1/event/energy_bias": 3.7353482246398926, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.867301940917969, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.470769882202148, "metrics/particle/momentum/class_1/event/px_bias": -0.5360336303710938, "metrics/particle/momentum/class_1/event/py_bias": -0.1658463031053543, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.747547626495361, "metrics/particle/momentum/class_1/event/pz_bias": 0.6322571039199829, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.675637245178223, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037506103515625, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19346827268600464, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19521097838878632, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.023426353931427002, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23538900911808014, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03292496129870415, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5683997869491577, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0854639783501625, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.409466505050659, "metrics/particle/momentum/class_2/accepted/samples": 1306.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8681623935699463, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7159651517868042, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7983066439628601, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.31644362211227417, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3461686670780182, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.295248985290527, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07508663833141327, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.952709197998047, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.12219087779521942, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.983925819396973, "metrics/particle/momentum/class_2/assigned/samples": 2597.0, "metrics/particle/momentum/class_2/efficiency": 0.44331297278404236, "metrics/particle/momentum/class_2/event/energy_bias": -8.617459297180176, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.66562271118164, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.565667629241943, "metrics/particle/momentum/class_2/event/px_bias": 0.4327540993690491, "metrics/particle/momentum/class_2/event/py_bias": -0.6855023503303528, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.344107627868652, "metrics/particle/momentum/class_2/event/pz_bias": -0.546515941619873, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.983844757080078, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023084789514541626, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1339114010334015, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1333424150943756, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.026755603030323982, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17504476010799408, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000858295417856425, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2043200135231018, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.02823769859969616, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7378730773925781, "metrics/particle/momentum/class_3/accepted/samples": 11651.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30428346991539, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.385884404182434, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3317451477050781, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.198233962059021, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15597398579120636, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.956718444824219, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007874015718698502, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49138033390045166, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.01353082712739706, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.9337784051895142, "metrics/particle/momentum/class_3/assigned/samples": 14605.0, "metrics/particle/momentum/class_3/efficiency": 0.7559693455696106, "metrics/particle/momentum/class_3/event/energy_bias": 3.4610989093780518, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.563922882080078, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.57050895690918, "metrics/particle/momentum/class_3/event/px_bias": -0.13461996614933014, "metrics/particle/momentum/class_3/event/py_bias": 0.15832774341106415, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.280527114868164, "metrics/particle/momentum/class_3/event/pz_bias": -1.583733320236206, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.982779502868652, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011258094571530819, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05483529716730118, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05548481270670891, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.011916069313883781, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10359402000904083, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23644863069057465, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0225333571434021, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7936411499977112, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11654265969991684, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.4452988803386688, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4455760419368744, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.39931121468544006, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.8773505687713623, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.37123697996139526, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.09552459418773651, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3853645324707031, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -0.5965687036514282, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3786585330963135, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9310078024864197, "metrics/particle/momentum/class_4/event/px_bias": 0.1454422026872635, "metrics/particle/momentum/class_4/event/py_bias": 0.086734339594841, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8376550078392029, "metrics/particle/momentum/class_4/event/pz_bias": -0.03668598458170891, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.8734169006347656, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005176492501050234, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01746579259634018, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01700509898364544, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.011668588034808636, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04972423240542412, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12943875789642334, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.041887786239385605, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5287810564041138, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.052278488874435425, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06960689276456833, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.059280212968587875, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04200766980648041, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.43514931201934814, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5913959741592407, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2082035094499588, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.427135944366455, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.23623444139957428, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8338087797164917, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5410898923873901, "metrics/particle/momentum/class_5/event/px_bias": 0.044086456298828125, "metrics/particle/momentum/class_5/event/py_bias": -0.000661943107843399, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5411816835403442, "metrics/particle/momentum/class_5/event/pz_bias": -0.13620972633361816, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3867197036743164, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.603661060333252, "step_valid_time": 13441.84378027916, "step_total_time": 13443.447441577911, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004922060507637, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05667583615440053, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2891, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2884, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588723051409619, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9553866490416392, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9571263035921205, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04461335095836087, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9565505804311775, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9530733641771315, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9548088064889919, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588723051409619, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003081290695892, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06621702079402247, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2238, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2141, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2138, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9502885042166, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9566577301161752, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9534624805165888, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04334226988382484, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9489569462938304, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553172475424486, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9521264751725673, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9502885042166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9982250044550292, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06831655720228968, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1189, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9268482490272374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.927570093457944, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.927209030751265, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07242990654205607, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9252918287937744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.92601246105919, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9256520046710782, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9268482490272374} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_32000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_32000.json new file mode 100644 index 0000000000000000000000000000000000000000..7d067f9ca9afdd8182f3e859d83d752675842ee4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_32000.json @@ -0,0 +1 @@ +{"epoch": 1.5160128861095319, "train": {"Classification_binary": 2.36964750289917, "Classification": 0.019903382286429405, "Regression_pt": 0.12665534019470215, "Regression_eta": 0.00888569001108408, "Regression_sin_phi": 0.008249124512076378, "Regression_cos_phi": 0.010563596151769161, "Regression_energy": 0.1287621706724167, "Total": 8.501825332641602, "Total_unweighted": 2.672666806727648}, "train_group": {}, "valid": {"Classification_binary": 2.4133655230204263, "Classification": 0.019273182998100918, "Regression_pt": 0.12416693568229675, "Regression_eta": 0.010220607121785482, "Regression_sin_phi": 0.007958260054389635, "Regression_cos_phi": 0.007944809893767038, "Regression_energy": 0.12757578492164612, "Total": 8.339533487955729, "Total_unweighted": 2.7105051036924124, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019732339307665825, "diagnostic/loss/clic_pf/Classification_binary": 2.417510986328125, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007495264522731304, "diagnostic/loss/clic_pf/Regression_energy": 0.13035213947296143, "diagnostic/loss/clic_pf/Regression_eta": 0.009355874732136726, "diagnostic/loss/clic_pf/Regression_pt": 0.1271272748708725, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007559174671769142, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03769668936729431, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18780308961868286, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05325031280517578, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42358413338661194, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.000549736840184778, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17975543439388275, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06939427554607391, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42305466532707214, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9716143012046814, "metrics/particle/class_1/pid_accuracy": 0.9874383807182312, "metrics/particle/class_2/efficiency": 0.4578258991241455, "metrics/particle/class_2/pid_accuracy": 0.7741379141807556, "metrics/particle/class_3/efficiency": 0.7525164484977722, "metrics/particle/class_3/pid_accuracy": 0.9851630926132202, "metrics/particle/class_4/efficiency": 0.940891444683075, "metrics/particle/class_4/pid_accuracy": 0.5509783625602722, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7052845358848572, "metrics/particle/count/bias_mean": -0.03202614560723305, "metrics/particle/count/mae": 2.9941177368164062, "metrics/particle/count/prediction_mean": 69.96732330322266, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04707404226064682, "metrics/particle/event/energy_response_mean": 1.0216155052185059, "metrics/particle/event/met_abs_error": 4.204728603363037, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036119554191827774, "metrics/particle/event/scalar_pt_response_mean": 1.0155582427978516, "metrics/particle/event/vector_pt_closure": 0.03652162849903107, "metrics/particle/matched/delta_eta_abs_mean": 0.008656763471662998, "metrics/particle/matched/delta_phi_abs_mean": 0.009560031816363335, "metrics/particle/matched/delta_r_mean": 0.014496472664177418, "metrics/particle/matched/energy_relative_abs_mean": 0.0838209018111229, "metrics/particle/matched/pid_accuracy": 0.9705536365509033, "metrics/particle/matched/pt_relative_abs_mean": 0.07339555025100708, "metrics/particle/matching/duplicate_fraction": 0.027501167729496956, "metrics/particle/matching/efficiency": 0.833948016166687, "metrics/particle/matching/f1": 0.8341388702392578, "metrics/particle/matching/purity": 0.8343297243118286, "metrics/particle/matching/target_coverage_dr0p05": 0.8122391700744629, "metrics/particle/matching/target_coverage_dr0p10": 0.8675618171691895, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.744189977645874, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.870969831943512, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01669607125222683, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07913074642419815, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07173630595207214, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01459532417356968, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1274595856666565, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003220241516828537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3103523552417755, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.00023535650689154863, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2240736484527588, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8695.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2829439640045166, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9481013417243958, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.9371014833450317, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8262850046157837, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11907029896974564, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.870429039001465, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.013240617699921131, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.701824426651001, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.009745062328875065, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.6415059566497803, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10498.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.808912456035614, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.7142302393913269, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.510403633117676, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.436471462249756, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22451956570148468, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.0009772591292858124, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.73491096496582, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.008139535784721375, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.76129150390625, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02719641476869583, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09040844440460205, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08577028661966324, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01703089103102684, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14182527363300323, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020744681358337402, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9121147394180298, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06628476083278656, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9756715297698975, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1880.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7379580140113831, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.597460389137268, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 8948.2705078125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 8948.1181640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24514563381671906, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 480370.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07489597797393799, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.6574223041534424, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.10320936888456345, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.987242221832275, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2884.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6119791865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": 1.1073424816131592, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.138089656829834, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0286973714828491, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.03440485894680023, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.051503583788871765, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.980645179748535, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.35369327664375305, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.419541358947754, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012378309853374958, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08526615798473358, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07295572757720947, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017274413257837296, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12990759313106537, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00036418501986190677, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10179203003644943, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010318992659449577, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5257174968719482, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19221.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1358400136232376, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.67620849609375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4879516065120697, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.39404118061065674, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08818227797746658, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.926982879638672, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0023570735938847065, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23304951190948486, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0014617594424635172, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4064433574676514, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21637.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.875512421131134, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.5273046493530273, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.398679256439209, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.480167388916016, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.1336788386106491, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3793148994445801, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.578308582305908, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.10565266758203506, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.951788902282715, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014573265798389912, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08380018919706345, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07340843975543976, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01647724024951458, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12998637557029724, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002483554882928729, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21378153562545776, "metrics/particle/momentum/all/accepted/pz_bias": -0.002405677456408739, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.083616018295288, "metrics/particle/momentum/all/accepted/samples": 29796.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22952647507190704, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.833586573600769, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 737.5198364257812, "metrics/particle/momentum/all/assigned/pt_relative_bias": 737.4161376953125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11036865413188934, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 137854.828125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011593705974519253, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5732387900352478, "metrics/particle/momentum/all/assigned/pz_bias": 0.012324383482336998, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.927090883255005, "metrics/particle/momentum/all/assigned/samples": 35019.0, "metrics/particle/momentum/all/efficiency": 0.8328720927238464, "metrics/particle/momentum/all/event/energy_bias": 3.348877191543579, "metrics/particle/momentum/all/event/p3_error_mean": 11.896574974060059, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.701157569885254, "metrics/particle/momentum/all/event/px_bias": 0.3926032781600952, "metrics/particle/momentum/all/event/py_bias": -0.3268340826034546, "metrics/particle/momentum/all/event/pz_abs_mean": 9.058897972106934, "metrics/particle/momentum/all/event/pz_bias": 0.2399010807275772, "metrics/particle/momentum/all/event/pz_residual_rms": 17.09963607788086, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006786690559238195, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.038111694157123566, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02032659202814102, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005106887314468622, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05889648571610451, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001281972974538803, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10709498077630997, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00744659686461091, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8665623664855957, "metrics/particle/momentum/class_1/accepted/samples": 16381.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.034925613552331924, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1521785706281662, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.12522678077220917, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.10035519301891327, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.018846677616238594, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 5.417295932769775, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0038523084949702024, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2177467793226242, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02973444014787674, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3807272911071777, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9708409905433655, "metrics/particle/momentum/class_1/event/energy_bias": 3.3827669620513916, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.687233924865723, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.374971389770508, "metrics/particle/momentum/class_1/event/px_bias": -0.16908235847949982, "metrics/particle/momentum/class_1/event/py_bias": -0.5379745960235596, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.6120100021362305, "metrics/particle/momentum/class_1/event/pz_bias": 0.677182137966156, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.583683967590332, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03793924301862717, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18945050239562988, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19107887148857117, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02601798251271248, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23228295147418976, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029695620760321617, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5332331657409668, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.024198975414037704, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.407538890838623, "metrics/particle/momentum/class_2/accepted/samples": 1347.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8934030532836914, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6806138753890991, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 9471.146484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 9470.6708984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3531571328639984, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 494276.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07562407851219177, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.989367723464966, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.062157709151506424, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.949916362762451, "metrics/particle/momentum/class_2/assigned/samples": 2724.0, "metrics/particle/momentum/class_2/efficiency": 0.4572301506996155, "metrics/particle/momentum/class_2/event/energy_bias": -5.417811870574951, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.358015060424805, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.387179374694824, "metrics/particle/momentum/class_2/event/px_bias": 0.6049796938896179, "metrics/particle/momentum/class_2/event/py_bias": -0.6595101952552795, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.07908821105957, "metrics/particle/momentum/class_2/event/pz_bias": -0.32943445444107056, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.677837371826172, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.023141996935009956, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13822290301322937, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13643090426921844, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03802900388836861, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17895002663135529, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011255411664023995, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2119351625442505, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01290653645992279, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7650132775306702, "metrics/particle/momentum/class_3/accepted/samples": 11550.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3343529999256134, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.625450849533081, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6935943365097046, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.56635582447052, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.17240220308303833, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 23.83902931213379, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009006586857140064, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5428324341773987, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.010957115329802036, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1609630584716797, "metrics/particle/momentum/class_3/assigned/samples": 14878.0, "metrics/particle/momentum/class_3/efficiency": 0.7494160532951355, "metrics/particle/momentum/class_3/event/energy_bias": 6.488086700439453, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.92232894897461, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.792915344238281, "metrics/particle/momentum/class_3/event/px_bias": -0.2531667947769165, "metrics/particle/momentum/class_3/event/py_bias": 0.7880064845085144, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.52383279800415, "metrics/particle/momentum/class_3/event/pz_bias": -0.01945635676383972, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.003239631652832, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01123481523245573, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05339257791638374, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05442788079380989, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004861961584538221, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1035766676068306, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22019679844379425, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.005326679442077875, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.771497905254364, "metrics/particle/momentum/class_4/accepted/samples": 340.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08911021053791046, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.604588270187378, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.342755913734436, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2919129133224487, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.0441988967359066, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.07076644897461, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2701426148414612, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.03014281578361988, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9226119518280029, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.939226508140564, "metrics/particle/momentum/class_4/event/energy_bias": -0.5689567923545837, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3725624084472656, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8976675868034363, "metrics/particle/momentum/class_4/event/px_bias": 0.12058316171169281, "metrics/particle/momentum/class_4/event/py_bias": 0.020092759281396866, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8712689876556396, "metrics/particle/momentum/class_4/event/pz_bias": 0.1350070983171463, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.187645196914673, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004709136672317982, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015645654872059822, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014842706732451916, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008191525936126709, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04927205666899681, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.15465308725833893, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05721733719110489, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6860520839691162, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.044481225311756134, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.02938850037753582, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03030993975698948, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013136055320501328, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11994808167219162, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.45669758319854736, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35993173718452454, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.091248035430908, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.5352084636688232, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.9268957376480103, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5885906219482422, "metrics/particle/momentum/class_5/event/px_bias": 0.089289590716362, "metrics/particle/momentum/class_5/event/py_bias": 0.06255142390727997, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.601357638835907, "metrics/particle/momentum/class_5/event/pz_bias": -0.2233973741531372, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.565256357192993, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6998052597045898, "step_valid_time": 13876.26528930664, "step_total_time": 13877.965095043182, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0042861316776572, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058283389341782454, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9611940298507463, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9548599670510708, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9580165289256198, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04514003294892916, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9538714991762768, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9570247933884297, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9611940298507463, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060865881510923, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06808301937609207, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2251, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.952063914780293, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9529098178587294, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9524866785079928, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047090182141270545, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507323568575233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9515770768547313, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511545293072824, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.952063914780293, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086589448506431, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06823711901910556, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1197, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1192, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315175097276265, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9300699300699301, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9307931570762052, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06993006993006994, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9276264591439689, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9261849261849262, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.926905132192846, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315175097276265} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_33000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_33000.json new file mode 100644 index 0000000000000000000000000000000000000000..63574c405fdac688b5c7893e3d7bab3707a9d2a2 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_33000.json @@ -0,0 +1 @@ +{"epoch": 1.5633882888004549, "train": {"Classification_binary": 2.395322799682617, "Classification": 0.017815958708524704, "Regression_pt": 0.12696054577827454, "Regression_eta": 0.009007620625197887, "Regression_sin_phi": 0.007306085899472237, "Regression_cos_phi": 0.009897853247821331, "Regression_energy": 0.12546370923519135, "Total": 8.30677318572998, "Total_unweighted": 2.6917745731770992}, "train_group": {}, "valid": {"Classification_binary": 2.3881284395853677, "Classification": 0.018885126958290737, "Regression_pt": 0.12122291326522827, "Regression_eta": 0.009933110947410265, "Regression_sin_phi": 0.007897483184933662, "Regression_cos_phi": 0.007932868475715319, "Regression_energy": 0.12441058953603108, "Total": 8.191588083902994, "Total_unweighted": 2.678410531952977, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019308503717184067, "diagnostic/loss/clic_pf/Classification_binary": 2.3869552612304688, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007578959222882986, "diagnostic/loss/clic_pf/Regression_energy": 0.12748722732067108, "diagnostic/loss/clic_pf/Regression_eta": 0.009349746629595757, "diagnostic/loss/clic_pf/Regression_pt": 0.12418682873249054, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007442953996360302, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02630983665585518, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18349745869636536, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04186346009373665, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4162696897983551, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.007878544740378857, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17606519162654877, "diagnostic/regression/clic_pf/pt_residual_mean": 0.060965992510318756, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41742050647735596, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.971514880657196, "metrics/particle/class_1/pid_accuracy": 0.9875189065933228, "metrics/particle/class_2/efficiency": 0.4583897292613983, "metrics/particle/class_2/pid_accuracy": 0.7803198099136353, "metrics/particle/class_3/efficiency": 0.7636167407035828, "metrics/particle/class_3/pid_accuracy": 0.9863102436065674, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.5568300485610962, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7317073345184326, "metrics/particle/count/bias_mean": -0.36535948514938354, "metrics/particle/count/mae": 3.0006535053253174, "metrics/particle/count/prediction_mean": 69.63398742675781, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04541376978158951, "metrics/particle/event/energy_response_mean": 1.0112872123718262, "metrics/particle/event/met_abs_error": 4.114693641662598, "metrics/particle/event/scalar_pt_relative_abs_error": 0.034655630588531494, "metrics/particle/event/scalar_pt_response_mean": 1.0087852478027344, "metrics/particle/event/vector_pt_closure": 0.03698762133717537, "metrics/particle/matched/delta_eta_abs_mean": 0.008619544096291065, "metrics/particle/matched/delta_phi_abs_mean": 0.008921327069401741, "metrics/particle/matched/delta_r_mean": 0.013989436440169811, "metrics/particle/matched/energy_relative_abs_mean": 0.08240967988967896, "metrics/particle/matched/pid_accuracy": 0.9716649055480957, "metrics/particle/matched/pt_relative_abs_mean": 0.0719866156578064, "metrics/particle/matching/duplicate_fraction": 0.02654402144253254, "metrics/particle/matching/efficiency": 0.8386446237564087, "metrics/particle/matching/f1": 0.8408389687538147, "metrics/particle/matching/purity": 0.8430448770523071, "metrics/particle/matching/target_coverage_dr0p05": 0.8195688128471375, "metrics/particle/matching/target_coverage_dr0p10": 0.868598222732544, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7508659958839417, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8739017248153687, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015406996943056583, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07925847172737122, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07227741181850433, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010637779720127583, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12792962789535522, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030804334674030542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.30736276507377625, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0365130715072155, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1613785028457642, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8765.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26877421140670776, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8584615588188171, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.875065803527832, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7624175548553467, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11539562791585922, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.8869805335998535, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01288536749780178, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.7130991220474243, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.07684002071619034, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.2829482555389404, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10477.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8154246807098389, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.6269518136978149, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.564094543457031, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.447380781173706, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.24942408502101898, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.21128572523593903, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.825831413269043, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.781355619430542, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.934054374694824, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022892499342560768, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0880776196718216, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08482135832309723, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019328411668539047, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1395290344953537, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01829587109386921, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8766306638717651, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.14948773384094238, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9915225505828857, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1913.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6845851540565491, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0876259803771973, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 1.7967489957809448, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.6474980115890503, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2299930453300476, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 9.972132682800293, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06854557991027832, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4772791862487793, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.18261650204658508, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.024399280548096, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2874.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6227213740348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.0927666574716568, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.892805576324463, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9754161834716797, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.018956564366817474, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.10729333013296127, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.756175518035889, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.9366496801376343, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.45029067993164, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.012616717256605625, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08450575172901154, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07198803871870041, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011977443471550941, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12829013168811798, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005693876300938427, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10621561855077744, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.00226353807374835, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5746647715568542, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19319.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13498802483081818, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6240732073783875, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.46681007742881775, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.36996427178382874, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08114492148160934, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.914621353149414, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002871566917747259, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22796818614006042, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.001989831915125251, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1437842845916748, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21591.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8799763321876526, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.845483660697937, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.4438982009887695, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.457196235656738, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.46786943078041077, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3943679928779602, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.701627731323242, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.43920421600341797, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.901495933532715, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014087343588471413, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08320031315088272, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07289101183414459, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01205479260534048, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12893146276474, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024335766211152077, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21412169933319092, "metrics/particle/momentum/all/accepted/pz_bias": 0.02166004851460457, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0851343870162964, "metrics/particle/momentum/all/accepted/samples": 29997.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2203071117401123, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7324796915054321, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6986095905303955, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5927152633666992, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10365749150514603, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 6.461836814880371, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011275828815996647, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.558436930179596, "metrics/particle/momentum/all/assigned/pz_bias": 0.03928954154253006, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8457682132720947, "metrics/particle/momentum/all/assigned/samples": 34942.0, "metrics/particle/momentum/all/efficiency": 0.8384905457496643, "metrics/particle/momentum/all/event/energy_bias": 1.3796687126159668, "metrics/particle/momentum/all/event/p3_error_mean": 11.784290313720703, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.690978527069092, "metrics/particle/momentum/all/event/px_bias": -0.23740190267562866, "metrics/particle/momentum/all/event/py_bias": -0.0757889524102211, "metrics/particle/momentum/all/event/pz_abs_mean": 9.033567428588867, "metrics/particle/momentum/all/event/pz_bias": 2.1572093963623047, "metrics/particle/momentum/all/event/pz_residual_rms": 15.32490348815918, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007350347936153412, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03837216645479202, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01998879574239254, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004290845710784197, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05919128656387329, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001404580194503069, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10886102169752121, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.01667473092675209, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8540800213813782, "metrics/particle/momentum/class_1/accepted/samples": 16375.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03733622655272484, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10163704305887222, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07982777059078217, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05423556640744209, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.01783915050327778, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7821358442306519, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0038523084949702024, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.19340434670448303, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02069830521941185, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.5386130809783936, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9704853892326355, "metrics/particle/momentum/class_1/event/energy_bias": 2.7788898944854736, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.011834144592285, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.280416011810303, "metrics/particle/momentum/class_1/event/px_bias": -0.607105553150177, "metrics/particle/momentum/class_1/event/py_bias": -0.286455363035202, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.996110439300537, "metrics/particle/momentum/class_1/event/pz_bias": 0.508548378944397, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.882060050964355, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.036264095455408096, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19174683094024658, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19438527524471283, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.030386174097657204, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23433861136436462, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02982848696410656, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5700628757476807, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.15033932030200958, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5233240127563477, "metrics/particle/momentum/class_2/accepted/samples": 1341.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8511996865272522, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6812886595726013, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7179911136627197, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.24358457326889038, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3400447368621826, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.597327470779419, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07233408093452454, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.945972204208374, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.19351212680339813, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.918541431427002, "metrics/particle/momentum/class_2/assigned/samples": 2682.0, "metrics/particle/momentum/class_2/efficiency": 0.45519348978996277, "metrics/particle/momentum/class_2/event/energy_bias": -5.639720916748047, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.41940689086914, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.588137149810791, "metrics/particle/momentum/class_2/event/px_bias": 0.49619609117507935, "metrics/particle/momentum/class_2/event/py_bias": -0.12633244693279266, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.948076248168945, "metrics/particle/momentum/class_2/event/pz_bias": 0.8964914083480835, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.226008415222168, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.021171867847442627, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13508720695972443, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1340823918581009, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027835490182042122, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17587614059448242, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008496898808516562, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2062365710735321, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.014252717606723309, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7387319207191467, "metrics/particle/momentum/class_3/accepted/samples": 11769.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3188004493713379, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.459857702255249, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4063467979431152, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2733440399169922, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16081655025482178, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.621883392333984, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008960452862083912, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5467616319656372, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03585858270525932, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7308883666992188, "metrics/particle/momentum/class_3/assigned/samples": 14843.0, "metrics/particle/momentum/class_3/efficiency": 0.7636257410049438, "metrics/particle/momentum/class_3/event/energy_bias": 5.154903888702393, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.731170654296875, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.630988121032715, "metrics/particle/momentum/class_3/event/px_bias": -0.2170194834470749, "metrics/particle/momentum/class_3/event/py_bias": 0.21388056874275208, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.411549091339111, "metrics/particle/momentum/class_3/event/pz_bias": 0.6983640193939209, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.202672958374023, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010586483404040337, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05240872502326965, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05300150811672211, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.009831610135734081, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09852585941553116, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2408028095960617, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.02556338533759117, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8493996858596802, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11811096966266632, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.0268480777740479, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.7006150484085083, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.6528083682060242, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 6.021145343780518, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.349809855222702, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.014790210872888565, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.137602686882019, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.5435001254081726, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3788979053497314, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8691800832748413, "metrics/particle/momentum/class_4/event/px_bias": 0.06763753294944763, "metrics/particle/momentum/class_4/event/py_bias": 0.06821499764919281, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9029748439788818, "metrics/particle/momentum/class_4/event/pz_bias": 0.18682023882865906, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3434622287750244, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004934446420520544, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01650043949484825, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01576503925025463, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006815973203629255, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0444638691842556, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1535329818725586, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.006717261392623186, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6118949055671692, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05697106942534447, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06472120434045792, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05604148656129837, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03122955933213234, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4365527033805847, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5839520692825317, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.1812623143196106, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3784894943237305, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.37090423703193665, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8438888788223267, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5568881034851074, "metrics/particle/momentum/class_5/event/px_bias": 0.02288951724767685, "metrics/particle/momentum/class_5/event/py_bias": 0.0549033023416996, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5291147232055664, "metrics/particle/momentum/class_5/event/pz_bias": -0.1330144703388214, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.287581205368042, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6227383613586426, "step_valid_time": 14309.49436044693, "step_total_time": 14311.117099046707, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991466420991284, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05731053557992838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9611940298507463, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9573835480673935, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9592850049652433, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04261645193260654, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9557317476048893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9576299238662694, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9611940298507463, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011817820210789, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06602063900739619, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2136, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2133, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.948069241011984, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9531459170013387, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9506008010680908, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04685408299866131, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9467376830892144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9518072289156626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9492656875834445, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.948069241011984, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0031839120403623, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06995621447300804, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1202, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06459143968871596, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.935408560311284} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_34000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_34000.json new file mode 100644 index 0000000000000000000000000000000000000000..e09226d390094e5ffc943708b87966fd8f837c73 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_34000.json @@ -0,0 +1 @@ +{"epoch": 1.6107636914913777, "train": {"Classification_binary": 2.3323874473571777, "Classification": 0.01901928335428238, "Regression_pt": 0.12696383893489838, "Regression_eta": 0.01049724780023098, "Regression_sin_phi": 0.007775556296110153, "Regression_cos_phi": 0.008389727212488651, "Regression_energy": 0.12880416214466095, "Total": 8.339839935302734, "Total_unweighted": 2.6338372630998492}, "train_group": {}, "valid": {"Classification_binary": 2.3804529507954917, "Classification": 0.018847811967134476, "Regression_pt": 0.12172158559163411, "Regression_eta": 0.010173552359143892, "Regression_sin_phi": 0.00793918160100778, "Regression_cos_phi": 0.007865041494369507, "Regression_energy": 0.12502984205881754, "Total": 8.208081563313803, "Total_unweighted": 2.672029965867599, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019342273473739624, "diagnostic/loss/clic_pf/Classification_binary": 2.3838820457458496, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007449951954185963, "diagnostic/loss/clic_pf/Regression_energy": 0.12760843336582184, "diagnostic/loss/clic_pf/Regression_eta": 0.009413914754986763, "diagnostic/loss/clic_pf/Regression_pt": 0.12432770431041718, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007620110176503658, "diagnostic/regression/clic_pf/energy_pred_mean": 0.029552767053246498, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18467268347740173, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04510639235377312, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4156765937805176, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0003481919120531529, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17805951833724976, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06919272243976593, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41843193769454956, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9723696708679199, "metrics/particle/class_1/pid_accuracy": 0.9874072670936584, "metrics/particle/class_2/efficiency": 0.45308977365493774, "metrics/particle/class_2/pid_accuracy": 0.7864609360694885, "metrics/particle/class_3/efficiency": 0.7590473294258118, "metrics/particle/class_3/pid_accuracy": 0.9866538047790527, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.54666668176651, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7617108225822449, "metrics/particle/count/bias_mean": -0.33137255907058716, "metrics/particle/count/mae": 2.9496731758117676, "metrics/particle/count/prediction_mean": 69.66797637939453, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04388120397925377, "metrics/particle/event/energy_response_mean": 1.0063396692276, "metrics/particle/event/met_abs_error": 4.222597122192383, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03457416594028473, "metrics/particle/event/scalar_pt_response_mean": 1.0102269649505615, "metrics/particle/event/vector_pt_closure": 0.03640057519078255, "metrics/particle/matched/delta_eta_abs_mean": 0.008802547119557858, "metrics/particle/matched/delta_phi_abs_mean": 0.00878793653100729, "metrics/particle/matched/delta_r_mean": 0.01402414869517088, "metrics/particle/matched/energy_relative_abs_mean": 0.08271096646785736, "metrics/particle/matched/pid_accuracy": 0.972070038318634, "metrics/particle/matched/pt_relative_abs_mean": 0.07289611548185349, "metrics/particle/matching/duplicate_fraction": 0.025940032675862312, "metrics/particle/matching/efficiency": 0.836767852306366, "metrics/particle/matching/f1": 0.8387531638145447, "metrics/particle/matching/purity": 0.8407478928565979, "metrics/particle/matching/target_coverage_dr0p05": 0.8184670209884644, "metrics/particle/matching/target_coverage_dr0p10": 0.8685515522956848, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7469537258148193, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8731734156608582, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.017452985048294067, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07815806567668915, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07169126719236374, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.018130144104361534, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1262136995792389, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002982677426189184, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3042813539505005, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0018436945974826813, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.166500210762024, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8717.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26528623700141907, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7432495951652527, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7745764851570129, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6674566268920898, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.12087072432041168, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.0089545249938965, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012411685660481453, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6643031239509583, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.048338375985622406, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.442885637283325, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10474.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8109591603279114, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.4342053532600403, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.4443583488464355, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.4909017086029053, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.45299893617630005, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.19629359245300293, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.630316257476807, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.1603962630033493, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.23308277130127, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023348307237029076, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0901138186454773, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0870252400636673, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.021440058946609497, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14261631667613983, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020558776333928108, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8973144292831421, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.0573158822953701, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.939183235168457, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1897.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.69398432970047, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.2093687057495117, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 9421.1845703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 9421.0322265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2319193333387375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 505109.09375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06641168147325516, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.500511407852173, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09345898032188416, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.98073673248291, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2876.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6175130009651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.7392067909240723, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.9406938552856445, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8956049680709839, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03832642361521721, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11136563122272491, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.8289642333984375, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.20791266858577728, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.825216293334961, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011657939292490482, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08412240445613861, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07211792469024658, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016659535467624664, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12837420403957367, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005184839246794581, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10109904408454895, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006717991549521685, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5849832892417908, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19287.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13613484799861908, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6113498210906982, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.44326257705688477, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3508060872554779, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08385782688856125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.2852067947387695, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.00254535349085927, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22782595455646515, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.002257999498397112, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4994783401489258, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21608.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8785187005996704, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4339157342910767, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.606046676635742, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.5860795974731445, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.05874146521091461, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.2756153345108032, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.7835474014282227, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.04768796265125275, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 7.89610481262207, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.014089030213654041, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08276373893022537, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07293929904699326, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017391547560691833, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12870290875434875, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025082773063331842, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21084657311439514, "metrics/particle/momentum/all/accepted/pz_bias": -0.0012345125433057547, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0795788764953613, "metrics/particle/momentum/all/accepted/samples": 29901.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.22072510421276093, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7000682950019836, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 775.5883178710938, "metrics/particle/momentum/all/assigned/pt_relative_bias": 775.4865112304688, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1071285530924797, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 144879.3125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010755764320492744, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5455759763717651, "metrics/particle/momentum/all/assigned/pz_bias": 0.020776167511940002, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.901339530944824, "metrics/particle/momentum/all/assigned/samples": 34958.0, "metrics/particle/momentum/all/efficiency": 0.8358071446418762, "metrics/particle/momentum/all/event/energy_bias": -0.7394963502883911, "metrics/particle/momentum/all/event/p3_error_mean": 12.002082824707031, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.837672233581543, "metrics/particle/momentum/all/event/px_bias": 0.3559311032295227, "metrics/particle/momentum/all/event/py_bias": 0.032043881714344025, "metrics/particle/momentum/all/event/pz_abs_mean": 9.165077209472656, "metrics/particle/momentum/all/event/pz_bias": -0.00017151236534118652, "metrics/particle/momentum/all/event/pz_residual_rms": 17.302583694458008, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.007361797150224447, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03851240873336792, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020070474594831467, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004745471756905317, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05835311487317085, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012211502762511373, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10542039573192596, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.001542140613310039, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8071937561035156, "metrics/particle/momentum/class_1/accepted/samples": 16378.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03864985331892967, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10536207258701324, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07764231413602829, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05204707384109497, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017840208485722542, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6269147396087646, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0034969178959727287, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2097342312335968, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.017758207395672798, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.294214963912964, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9706631898880005, "metrics/particle/momentum/class_1/event/energy_bias": 1.4091150760650635, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.923464775085449, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.132053852081299, "metrics/particle/momentum/class_1/event/px_bias": -0.3412720561027527, "metrics/particle/momentum/class_1/event/py_bias": -0.37349897623062134, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.955646991729736, "metrics/particle/momentum/class_1/event/pz_bias": 0.6918857097625732, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.222745895385742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.037747710943222046, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19196036458015442, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1921193152666092, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.024072740226984024, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23399914801120758, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031111111864447594, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5871654748916626, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.022148828953504562, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.642723798751831, "metrics/particle/momentum/class_2/accepted/samples": 1350.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8831057548522949, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6322095394134521, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 9977.8994140625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 9977.423828125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34732964634895325, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 519869.96875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07403314858675003, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.966099977493286, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1392953097820282, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.370203495025635, "metrics/particle/momentum/class_2/assigned/samples": 2715.0, "metrics/particle/momentum/class_2/efficiency": 0.45824846625328064, "metrics/particle/momentum/class_2/event/energy_bias": -6.049229145050049, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.39468002319336, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.5073957443237305, "metrics/particle/momentum/class_2/event/px_bias": 0.7078715562820435, "metrics/particle/momentum/class_2/event/py_bias": -0.1258048564195633, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.065346717834473, "metrics/particle/momentum/class_2/event/pz_bias": -0.07701125741004944, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.884432792663574, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.0210214015096426, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1342592090368271, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1349295824766159, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04048575833439827, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17628388106822968, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011154011590406299, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1999158412218094, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0066873072646558285, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7169243693351746, "metrics/particle/momentum/class_3/accepted/samples": 11655.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31240034103393555, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3750203847885132, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4273364543914795, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3049732446670532, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16739732027053833, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.476045608520508, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007688676007091999, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4921802878379822, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.007819971069693565, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 1.9309412240982056, "metrics/particle/momentum/class_3/assigned/samples": 14827.0, "metrics/particle/momentum/class_3/efficiency": 0.7562289237976074, "metrics/particle/momentum/class_3/event/energy_bias": 4.755107879638672, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.13791275024414, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.466032028198242, "metrics/particle/momentum/class_3/event/px_bias": -0.1766113042831421, "metrics/particle/momentum/class_3/event/py_bias": 0.40079057216644287, "metrics/particle/momentum/class_3/event/pz_abs_mean": 5.904120922088623, "metrics/particle/momentum/class_3/event/pz_bias": -0.520246684551239, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.311479568481445, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011165820062160492, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.050265271216630936, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05102968588471413, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007661753799766302, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09412980824708939, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22965075075626373, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.010921534150838852, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7810677289962769, "metrics/particle/momentum/class_4/accepted/samples": 341.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08213687688112259, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6165640354156494, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.495226263999939, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4485570192337036, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 24.15443992614746, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2628614604473114, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.010883438400924206, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8177492022514343, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9419889450073242, "metrics/particle/momentum/class_4/event/energy_bias": -0.6326987743377686, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.1560864448547363, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.7603301405906677, "metrics/particle/momentum/class_4/event/px_bias": 0.07063990831375122, "metrics/particle/momentum/class_4/event/py_bias": 0.06618797034025192, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.718403697013855, "metrics/particle/momentum/class_4/event/pz_bias": 0.048997681587934494, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.548985481262207, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0052722143009305, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.016296325251460075, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.0162558201700449, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0018496493576094508, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.047087401151657104, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1522306650876999, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.058793503791093826, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6099006533622742, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.025730151683092117, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03432217240333557, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.034464020282030106, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010862301103770733, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12429851293563843, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.48315852880477905, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.39228829741477966, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.6767683029174805, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.22179073095321655, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.936030387878418, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.6143351793289185, "metrics/particle/momentum/class_5/event/px_bias": 0.0953029990196228, "metrics/particle/momentum/class_5/event/py_bias": 0.06436912715435028, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5929400324821472, "metrics/particle/momentum/class_5/event/pz_bias": -0.14379699528217316, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4424445629119873, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5989534854888916, "step_valid_time": 14744.825238704681, "step_total_time": 14746.424192428589, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025076563462378, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05731534209600109, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3017, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2887, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9592039800995025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9585681140205502, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9588859416445623, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04143188597944979, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9575456053067993, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9569108385813723, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9572281167108754, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9592039800995025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050965228301494, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06586709289508486, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2254, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2152, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2146, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9551708832667555, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9547471162377995, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549589527401819, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04525288376220053, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9525077674212161, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9520851818988465, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522964277790105, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9551708832667555, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0073802095254125, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06459393052250362, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1202, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1200, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.933954933954934, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9346811819595645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06604506604506605, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.933852140077821, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9324009324009324, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9331259720062208, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.935408560311284} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_35000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_35000.json new file mode 100644 index 0000000000000000000000000000000000000000..5440b9c5955d72acba9326f784b9ed1ca80d0d1e --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_35000.json @@ -0,0 +1 @@ +{"epoch": 1.6581390941823007, "train": {"Classification_binary": 2.3414499759674072, "Classification": 0.019064409658312798, "Regression_pt": 0.12574903666973114, "Regression_eta": 0.008780544623732567, "Regression_sin_phi": 0.007505933288484812, "Regression_cos_phi": 0.008094127289950848, "Regression_energy": 0.12656281888484955, "Total": 8.182061195373535, "Total_unweighted": 2.637206846382469}, "train_group": {}, "valid": {"Classification_binary": 2.381911595662435, "Classification": 0.018964752554893494, "Regression_pt": 0.12209101517995198, "Regression_eta": 0.01005467213690281, "Regression_sin_phi": 0.008013378207882246, "Regression_cos_phi": 0.007929022734363874, "Regression_energy": 0.12556511163711548, "Total": 8.230062484741211, "Total_unweighted": 2.674529548113545, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019458625465631485, "diagnostic/loss/clic_pf/Classification_binary": 2.3916172981262207, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007513869553804398, "diagnostic/loss/clic_pf/Regression_energy": 0.1290428340435028, "diagnostic/loss/clic_pf/Regression_eta": 0.009286046959459782, "diagnostic/loss/clic_pf/Regression_pt": 0.1257765293121338, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007535114418715239, "diagnostic/regression/clic_pf/energy_pred_mean": 0.036710213869810104, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18642273545265198, "diagnostic/regression/clic_pf/energy_residual_mean": 0.052263833582401276, "diagnostic/regression/clic_pf/energy_residual_rms": 0.42174598574638367, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0033066582400351763, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17959648370742798, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07215119153261185, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4233686625957489, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9720714688301086, "metrics/particle/class_1/pid_accuracy": 0.9867285490036011, "metrics/particle/class_2/efficiency": 0.45523229241371155, "metrics/particle/class_2/pid_accuracy": 0.7822640538215637, "metrics/particle/class_3/efficiency": 0.7535079121589661, "metrics/particle/class_3/pid_accuracy": 0.9861552119255066, "metrics/particle/class_4/efficiency": 0.9350775480270386, "metrics/particle/class_4/pid_accuracy": 0.5492228269577026, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.7361963391304016, "metrics/particle/count/bias_mean": -0.7183006405830383, "metrics/particle/count/mae": 3.0333333015441895, "metrics/particle/count/prediction_mean": 69.28104400634766, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.046040717512369156, "metrics/particle/event/energy_response_mean": 1.0047342777252197, "metrics/particle/event/met_abs_error": 4.119125843048096, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03456712141633034, "metrics/particle/event/scalar_pt_response_mean": 1.006929636001587, "metrics/particle/event/vector_pt_closure": 0.03644397109746933, "metrics/particle/matched/delta_eta_abs_mean": 0.007305334787815809, "metrics/particle/matched/delta_phi_abs_mean": 0.00912819616496563, "metrics/particle/matched/delta_r_mean": 0.013109353370964527, "metrics/particle/matched/energy_relative_abs_mean": 0.08246634900569916, "metrics/particle/matched/pid_accuracy": 0.97117018699646, "metrics/particle/matched/pt_relative_abs_mean": 0.07272816449403763, "metrics/particle/matching/duplicate_fraction": 0.02559433877468109, "metrics/particle/matching/efficiency": 0.8342934846878052, "metrics/particle/matching/f1": 0.8385961651802063, "metrics/particle/matching/purity": 0.8429433703422546, "metrics/particle/matching/target_coverage_dr0p05": 0.8201010227203369, "metrics/particle/matching/target_coverage_dr0p10": 0.8676645159721375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7453384399414062, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.869503915309906, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015541348606348038, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.077898308634758, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07146026194095612, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014801083132624626, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12582527101039886, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003447483293712139, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3011326789855957, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.004282137379050255, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1469088792800903, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8702.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2593604326248169, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.6539695262908936, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6689819097518921, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.559121310710907, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1166074350476265, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.801530838012695, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011142061091959476, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6429298520088196, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.04029296338558197, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4756224155426025, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10411.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8095636963844299, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.790215253829956, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.105794906616211, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2828145027160645, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.07954016327857971, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.044627595692873, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.364035606384277, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.4189082384109497, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.296568870544434, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02393462508916855, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08674375712871552, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08296027779579163, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.0180570837110281, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13711467385292053, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020299145951867104, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8750519156455994, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.13235774636268616, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8999440670013428, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1872.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6923404335975647, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1919443607330322, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 23362.064453125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 23361.912109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23985891044139862, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1243751.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06525573134422302, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4968421459198, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.09165404736995697, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.888506889343262, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2835.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.609375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.985215663909912, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.352112770080566, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9749383926391602, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04000488668680191, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12932761013507843, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.225266933441162, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.14431527256965637, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.25577735900879, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011210326105356216, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08453267067670822, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07280779629945755, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.017453692853450775, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12912045419216156, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005195075063966215, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1023024171590805, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.011027790606021881, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5497741103172302, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19249.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13371503353118896, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5616653561592102, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.43714383244514465, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3469325602054596, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0837353840470314, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.10715389251709, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026906661223620176, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23110458254814148, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0026187461335211992, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.142181634902954, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21556.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8767878413200378, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.508197546005249, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 6.11000919342041, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.9740777015686035, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.7247711420059204, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3748835623264313, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 3.098240375518799, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.09571835398674011, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 12.92406940460205, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013272775337100029, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08273563534021378, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07305187731981277, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016717568039894104, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12868784368038177, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026154310908168554, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2088245004415512, "metrics/particle/momentum/all/accepted/pz_bias": -5.912752021686174e-05, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0520371198654175, "metrics/particle/momentum/all/accepted/samples": 29823.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21680788695812225, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6406211853027344, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1903.5643310546875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1903.462890625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10628699511289597, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 354983.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010315499268472195, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5388708114624023, "metrics/particle/momentum/all/assigned/pz_bias": 0.021141858771443367, "metrics/particle/momentum/all/assigned/pz_residual_rms": 3.121687650680542, "metrics/particle/momentum/all/assigned/samples": 34802.0, "metrics/particle/momentum/all/efficiency": 0.8336268067359924, "metrics/particle/momentum/all/event/energy_bias": -1.2672333717346191, "metrics/particle/momentum/all/event/p3_error_mean": 12.756722450256348, "metrics/particle/momentum/all/event/pt_vector_error_mean": 6.1784749031066895, "metrics/particle/momentum/all/event/px_bias": -0.685235857963562, "metrics/particle/momentum/all/event/py_bias": -0.29018354415893555, "metrics/particle/momentum/all/event/pz_abs_mean": 9.726521492004395, "metrics/particle/momentum/all/event/pz_bias": 0.37031134963035583, "metrics/particle/momentum/all/event/pz_residual_rms": 20.606449127197266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0065973857417702675, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03752909228205681, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.020370354875922203, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.005722628440707922, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05795727297663689, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015270905569195747, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10351778566837311, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00017571001080796123, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8129630088806152, "metrics/particle/momentum/class_1/accepted/samples": 16371.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03794470429420471, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10971496254205704, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10309651494026184, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07853242009878159, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017779884859919548, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.597277879714966, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089373629540205, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.21725265681743622, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.02748682163655758, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.921168565750122, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9702483415603638, "metrics/particle/momentum/class_1/event/energy_bias": 1.8715358972549438, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.910701751708984, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.709297180175781, "metrics/particle/momentum/class_1/event/px_bias": -0.9105589389801025, "metrics/particle/momentum/class_1/event/py_bias": -0.46199747920036316, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.678098678588867, "metrics/particle/momentum/class_1/event/pz_bias": 0.6019178628921509, "metrics/particle/momentum/class_1/event/pz_residual_rms": 18.151321411132812, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03575359657406807, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19102585315704346, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1919955462217331, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.0322796069085598, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23276673257350922, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029147982597351074, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5111583471298218, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07455077022314072, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.3568239212036133, "metrics/particle/momentum/class_2/accepted/samples": 1338.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.860224187374115, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6337235569953918, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 24934.115234375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 24933.640625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34939759969711304, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1284979.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06890060007572174, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.884413003921509, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.04833618924021721, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.780635356903076, "metrics/particle/momentum/class_2/assigned/samples": 2656.0, "metrics/particle/momentum/class_2/efficiency": 0.4541751444339752, "metrics/particle/momentum/class_2/event/energy_bias": -7.312556266784668, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.462207794189453, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.452506065368652, "metrics/particle/momentum/class_2/event/px_bias": 0.10430392622947693, "metrics/particle/momentum/class_2/event/py_bias": -0.6337949633598328, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.149636268615723, "metrics/particle/momentum/class_2/event/pz_bias": 0.6882757544517517, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.113957405090332, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020300254225730896, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1357707977294922, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1349475383758545, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03810635954141617, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17662668228149414, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012067925417795777, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20717601478099823, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008264887146651745, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7638852596282959, "metrics/particle/momentum/class_3/accepted/samples": 11601.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31131619215011597, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2297710180282593, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3919408321380615, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2673380374908447, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16613483428955078, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.62785530090332, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0071287937462329865, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4898594915866852, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.015200991183519363, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1698708534240723, "metrics/particle/momentum/class_3/assigned/samples": 14729.0, "metrics/particle/momentum/class_3/efficiency": 0.7527251243591309, "metrics/particle/momentum/class_3/event/energy_bias": 4.840198516845703, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.504711151123047, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.517279624938965, "metrics/particle/momentum/class_3/event/px_bias": -0.2420044243335724, "metrics/particle/momentum/class_3/event/py_bias": 0.7041062116622925, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.2714056968688965, "metrics/particle/momentum/class_3/event/pz_bias": -0.7643712759017944, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.399408340454102, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011078623123466969, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05771784111857414, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05856107920408249, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.014703452587127686, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.1088273897767067, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24538305401802063, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.00412727752700448, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.828879177570343, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07981590926647186, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.7519406080245972, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6182764768600464, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.5712486505508423, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.77999496459961, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3111518323421478, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.012935484759509563, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.969103217124939, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.24564561247825623, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.5480518341064453, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.013930320739746, "metrics/particle/momentum/class_4/event/px_bias": 0.2812656760215759, "metrics/particle/momentum/class_4/event/py_bias": 0.07846732437610626, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9781175851821899, "metrics/particle/momentum/class_4/event/pz_bias": 0.046021416783332825, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.9753854274749756, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004317556507885456, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.016797160729765892, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01723257452249527, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005992967635393143, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.05344952270388603, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1426861733198166, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05590203031897545, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.7905358076095581, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.033417679369449615, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07146286219358444, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06266125291585922, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.04098649322986603, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4304354786872864, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5455976128578186, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.415383905172348, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.29537296295166, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.42076602578163147, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7896188497543335, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5105542540550232, "metrics/particle/momentum/class_5/event/px_bias": 0.08175792545080185, "metrics/particle/momentum/class_5/event/py_bias": 0.023035356774926186, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5157111287117004, "metrics/particle/momentum/class_5/event/pz_bias": -0.20153242349624634, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3112266063690186, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6628167629241943, "step_valid_time": 15178.582605361938, "step_total_time": 15180.245422363281, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029058849103007, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.058287133988192075, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3017, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9601990049751243, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.959562479284057, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9598806366047745, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04043752071594299, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9565793834935367, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9568965517241379, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9601990049751243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001947225779423, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06543169701523999, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529516200621394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9559216384683883, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9544343187374972, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04407836153161175, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9545859305431879, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9531006890420093, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529516200621394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023599572295563, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06736618969606811, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1202, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9369649805447471, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362363919129082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9366005445352004, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06376360808709176, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9346811819595645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9350447296771685, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9369649805447471} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_36000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_36000.json new file mode 100644 index 0000000000000000000000000000000000000000..b5d78eaceafd7e4dc6ac23eef2b5b101d69443b5 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_36000.json @@ -0,0 +1 @@ +{"epoch": 1.7055144968732234, "train": {"Classification_binary": 2.307271718978882, "Classification": 0.017644254490733147, "Regression_pt": 0.115592360496521, "Regression_eta": 0.011071735061705112, "Regression_sin_phi": 0.0066658686846494675, "Regression_cos_phi": 0.006613907404243946, "Regression_energy": 0.11760438233613968, "Total": 7.777561187744141, "Total_unweighted": 2.582464227452874}, "train_group": {}, "valid": {"Classification_binary": 2.3608241081237793, "Classification": 0.018672445168097813, "Regression_pt": 0.11998780568440755, "Regression_eta": 0.010007593159874281, "Regression_sin_phi": 0.007894774278004965, "Regression_cos_phi": 0.007865021005272865, "Regression_energy": 0.12319675087928772, "Total": 8.118983586629232, "Total_unweighted": 2.6484484982987246, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01923091895878315, "diagnostic/loss/clic_pf/Classification_binary": 2.363013744354248, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007404041476547718, "diagnostic/loss/clic_pf/Regression_energy": 0.12533603608608246, "diagnostic/loss/clic_pf/Regression_eta": 0.009272534400224686, "diagnostic/loss/clic_pf/Regression_pt": 0.122263103723526, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007426553405821323, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03444632142782211, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18346428871154785, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04999994486570358, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4182777404785156, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.0027110865339636803, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1765374392271042, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07155562192201614, "diagnostic/regression/clic_pf/pt_residual_rms": 0.42014285922050476, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9727672338485718, "metrics/particle/class_1/pid_accuracy": 0.988842785358429, "metrics/particle/class_2/efficiency": 0.45771312713623047, "metrics/particle/class_2/pid_accuracy": 0.7915742993354797, "metrics/particle/class_3/efficiency": 0.7597585916519165, "metrics/particle/class_3/pid_accuracy": 0.9865812063217163, "metrics/particle/class_4/efficiency": 0.9399224519729614, "metrics/particle/class_4/pid_accuracy": 0.5463917255401611, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7317073345184326, "metrics/particle/count/bias_mean": -0.1993464082479477, "metrics/particle/count/mae": 2.93137264251709, "metrics/particle/count/prediction_mean": 69.80000305175781, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04487886652350426, "metrics/particle/event/energy_response_mean": 1.0121996402740479, "metrics/particle/event/met_abs_error": 4.058459281921387, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03601368889212608, "metrics/particle/event/scalar_pt_response_mean": 1.0143141746520996, "metrics/particle/event/vector_pt_closure": 0.036234017461538315, "metrics/particle/matched/delta_eta_abs_mean": 0.008210749365389347, "metrics/particle/matched/delta_phi_abs_mean": 0.00889502838253975, "metrics/particle/matched/delta_r_mean": 0.013624605722725391, "metrics/particle/matched/energy_relative_abs_mean": 0.08218561857938766, "metrics/particle/matched/pid_accuracy": 0.9728337526321411, "metrics/particle/matched/pt_relative_abs_mean": 0.07230391353368759, "metrics/particle/matching/duplicate_fraction": 0.02670562081038952, "metrics/particle/matching/efficiency": 0.8376082181930542, "metrics/particle/matching/f1": 0.8388025760650635, "metrics/particle/matching/purity": 0.8400003910064697, "metrics/particle/matching/target_coverage_dr0p05": 0.8192606568336487, "metrics/particle/matching/target_coverage_dr0p10": 0.8702882528305054, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7489892244338989, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.873733639717102, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01585274748504162, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07845059782266617, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07243027538061142, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.019006695598363876, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12794581055641174, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028574694879353046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3029237389564514, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.020380014553666115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1577351093292236, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8749.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2590039074420929, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.850350558757782, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8164827227592468, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7125552892684937, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11785442382097244, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 8.219597816467285, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012195121496915817, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6902570128440857, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02849181927740574, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1979236602783203, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10496.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.813936173915863, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.8480234742164612, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.442540168762207, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.419586181640625, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.19336169958114624, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.13060343265533447, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.687738418579102, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3702400326728821, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.379532814025879, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02420443296432495, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0868278220295906, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08602148294448853, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02533535473048687, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14116233587265015, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021164020523428917, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8956377506256104, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05623101070523262, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0662450790405273, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1890.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6943989992141724, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5812203884124756, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 38942.59765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 38942.45703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.24287699162960052, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2088971.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0722724124789238, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.5188703536987305, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.04649125039577484, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.152577877044678, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2878.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.615234375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4338512718677521, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.77490234375, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9984917044639587, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09159533679485321, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08507929742336273, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.608823299407959, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.20553413033485413, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.2742280960083, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011760946363210678, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08382213115692139, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07142411917448044, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01610642857849598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1278541088104248, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005697120213881135, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10140863806009293, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007884790189564228, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5616827011108398, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19308.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13178008794784546, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6573028564453125, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.46765246987342834, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3762124180793762, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08179797977209091, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.071499347686768, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002777520567178726, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22534051537513733, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.014373301528394222, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4154478311538696, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21602.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8794752955436707, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.3664801120758057, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.211745738983154, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.25480318069458, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.4878506362438202, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.29192298650741577, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.574195623397827, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.15546241402626038, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 8.089079856872559, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013741692528128624, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08244252949953079, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0726393312215805, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.017536191269755363, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12876130640506744, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025378167629241943, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2104061394929886, "metrics/particle/momentum/all/accepted/pz_bias": -0.007488819304853678, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0901169776916504, "metrics/particle/momentum/all/accepted/samples": 29947.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21625398099422455, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7912594079971313, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3204.925537109375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3204.82568359375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10587260127067566, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 599228.9375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.011322049424052238, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5535811185836792, "metrics/particle/momentum/all/assigned/pz_bias": 0.0034983970690518618, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9182190895080566, "metrics/particle/momentum/all/assigned/samples": 34976.0, "metrics/particle/momentum/all/efficiency": 0.837092936038971, "metrics/particle/momentum/all/event/energy_bias": 1.780652403831482, "metrics/particle/momentum/all/event/p3_error_mean": 11.87061882019043, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.627429008483887, "metrics/particle/momentum/all/event/px_bias": -0.38608428835868835, "metrics/particle/momentum/all/event/py_bias": -0.3374471068382263, "metrics/particle/momentum/all/event/pz_abs_mean": 9.170215606689453, "metrics/particle/momentum/all/event/pz_bias": -0.7312366366386414, "metrics/particle/momentum/all/event/pz_residual_rms": 16.181289672851562, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006981272250413895, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03792521357536316, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.019541356712579727, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004342865198850632, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.058492206037044525, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0015856559621170163, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10504602640867233, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0029753916896879673, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8488460779190063, "metrics/particle/momentum/class_1/accepted/samples": 16397.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.034824665635824203, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10803739726543427, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09278576076030731, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06792096793651581, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017484590411186218, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.962164282798767, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004089615773409605, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.191266268491745, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.018744206055998802, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.4050661325454712, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9717892408370972, "metrics/particle/momentum/class_1/event/energy_bias": 3.267080545425415, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.905280113220215, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.100928783416748, "metrics/particle/momentum/class_1/event/px_bias": -0.4849405586719513, "metrics/particle/momentum/class_1/event/py_bias": -0.46775341033935547, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.990069389343262, "metrics/particle/momentum/class_1/event/pz_bias": -0.03692662715911865, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.669565200805664, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03596791625022888, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19386844336986542, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1948668360710144, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019190724939107895, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23510538041591644, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031804732978343964, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.613006830215454, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04004838317632675, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.620483636856079, "metrics/particle/momentum/class_2/accepted/samples": 1352.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8598419427871704, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7321546077728271, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 41369.25, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 41368.79296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34256184101104736, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2153145.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07788851857185364, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.0168535709381104, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.03397567570209503, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.4052252769470215, "metrics/particle/momentum/class_2/assigned/samples": 2709.0, "metrics/particle/momentum/class_2/efficiency": 0.45892736315727234, "metrics/particle/momentum/class_2/event/energy_bias": -5.788098335266113, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.299112319946289, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.365981578826904, "metrics/particle/momentum/class_2/event/px_bias": 0.362259179353714, "metrics/particle/momentum/class_2/event/py_bias": -0.3695186674594879, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.108359336853027, "metrics/particle/momentum/class_2/event/pz_bias": -0.5713037848472595, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.910211563110352, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02086540311574936, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13383647799491882, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1343943029642105, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04076847434043884, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1759672313928604, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0005991611978970468, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19622452557086945, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.018256237730383873, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7032369375228882, "metrics/particle/momentum/class_3/accepted/samples": 11683.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30966582894325256, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5552427768707275, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.6119849681854248, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4909043312072754, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16557808220386505, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.653268814086914, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007676250766962767, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5212879180908203, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.026616444811224937, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.7472012042999268, "metrics/particle/momentum/class_3/assigned/samples": 14851.0, "metrics/particle/momentum/class_3/efficiency": 0.7580456733703613, "metrics/particle/momentum/class_3/event/energy_bias": 5.264649868011475, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.424076080322266, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.577281475067139, "metrics/particle/momentum/class_3/event/px_bias": -0.515326738357544, "metrics/particle/momentum/class_3/event/py_bias": 0.4493982791900635, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.128570556640625, "metrics/particle/momentum/class_3/event/pz_bias": -0.09868019819259644, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.345754623413086, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011032084003090858, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05536136403679848, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.055473022162914276, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.00873374193906784, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10336741805076599, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24474439024925232, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.014641829766333103, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.8182936906814575, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10325001180171967, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.1014013290405273, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.8161078691482544, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.76625394821167, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05248618870973587, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.793840408325195, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.322385311126709, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02102508395910263, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9721460938453674, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -0.5932531356811523, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3333640098571777, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8820145130157471, "metrics/particle/momentum/class_4/event/px_bias": 0.18236538767814636, "metrics/particle/momentum/class_4/event/py_bias": 0.024892376735806465, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8281491994857788, "metrics/particle/momentum/class_4/event/pz_bias": 0.16357669234275818, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.13482403755188, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.005244845058768988, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.014989806339144707, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014761668629944324, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0036566287744790316, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.042131293565034866, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12830248475074768, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06406570225954056, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5176766514778137, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05825195834040642, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06185917183756828, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.05163775011897087, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.026560435071587563, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4022231101989746, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5714331269264221, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.22581106424331665, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.427737712860107, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.36972665786743164, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7599221467971802, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5200924277305603, "metrics/particle/momentum/class_5/event/px_bias": 0.06955838948488235, "metrics/particle/momentum/class_5/event/py_bias": 0.02553429827094078, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.46395573019981384, "metrics/particle/momentum/class_5/event/pz_bias": -0.18790271878242493, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2320117950439453, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6286046504974365, "step_valid_time": 15614.325213432312, "step_total_time": 15615.953818321228, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018373343281022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05834545571033667, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9618573797678275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958994708994709, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9604239112435834, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041005291005291, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957010582010582, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9584368272892863, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9618573797678275, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004534664786592, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06940688058874667, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2250, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2150, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.954283177984909, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9555555555555556, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9549189429269376, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044444444444444446, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533954727030626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9546666666666667, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9540306462358428, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.954283177984909, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0091736208494366, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06552991954889076, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1288, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1197, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9309006211180124, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9319860085503303, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06909937888198758, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315175097276265, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9293478260869565, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9304314030314808, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9330739299610895} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_37000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_37000.json new file mode 100644 index 0000000000000000000000000000000000000000..3bfb7b10c393334e17eebe544a3a1d76289352ae --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_37000.json @@ -0,0 +1 @@ +{"epoch": 1.7528898995641462, "train": {"Classification_binary": 2.3142902851104736, "Classification": 0.01895718090236187, "Regression_pt": 0.1323545277118683, "Regression_eta": 0.009922821074724197, "Regression_sin_phi": 0.007532597053796053, "Regression_cos_phi": 0.006993005983531475, "Regression_energy": 0.13456042110919952, "Total": 8.344547271728516, "Total_unweighted": 2.624610838945955}, "train_group": {}, "valid": {"Classification_binary": 2.3587118784586587, "Classification": 0.018617256234089535, "Regression_pt": 0.11951483289400737, "Regression_eta": 0.009868312006195387, "Regression_sin_phi": 0.007809074595570564, "Regression_cos_phi": 0.0077759865671396255, "Regression_energy": 0.12270760536193848, "Total": 8.083003362019857, "Total_unweighted": 2.6450049461175995, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019034888595342636, "diagnostic/loss/clic_pf/Classification_binary": 2.3634033203125, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007287443149834871, "diagnostic/loss/clic_pf/Regression_energy": 0.12565700709819794, "diagnostic/loss/clic_pf/Regression_eta": 0.009202963672578335, "diagnostic/loss/clic_pf/Regression_pt": 0.12261150032281876, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.0072733317501842976, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03295142576098442, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1835174858570099, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04850504919886589, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4136989414691925, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.001355227897875011, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17568708956241608, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06748931109905243, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41529250144958496, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9739598631858826, "metrics/particle/class_1/pid_accuracy": 0.9873257279396057, "metrics/particle/class_2/efficiency": 0.4529770016670227, "metrics/particle/class_2/pid_accuracy": 0.7796863317489624, "metrics/particle/class_3/efficiency": 0.7539605498313904, "metrics/particle/class_3/pid_accuracy": 0.9870783090591431, "metrics/particle/class_4/efficiency": 0.9418604373931885, "metrics/particle/class_4/pid_accuracy": 0.5699588656425476, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.772357702255249, "metrics/particle/count/bias_mean": -0.9764705896377563, "metrics/particle/count/mae": 3.1098039150238037, "metrics/particle/count/prediction_mean": 69.02287292480469, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.044366009533405304, "metrics/particle/event/energy_response_mean": 1.0084835290908813, "metrics/particle/event/met_abs_error": 3.992353677749634, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0342235267162323, "metrics/particle/event/scalar_pt_response_mean": 1.0087794065475464, "metrics/particle/event/vector_pt_closure": 0.03563416004180908, "metrics/particle/matched/delta_eta_abs_mean": 0.007648389786481857, "metrics/particle/matched/delta_phi_abs_mean": 0.00839183758944273, "metrics/particle/matched/delta_r_mean": 0.012814516201615334, "metrics/particle/matched/energy_relative_abs_mean": 0.08275283128023148, "metrics/particle/matched/pid_accuracy": 0.9721880555152893, "metrics/particle/matched/pt_relative_abs_mean": 0.07177489250898361, "metrics/particle/matching/duplicate_fraction": 0.02527342364192009, "metrics/particle/matching/efficiency": 0.8352832198143005, "metrics/particle/matching/f1": 0.841150164604187, "metrics/particle/matching/purity": 0.847100019454956, "metrics/particle/matching/target_coverage_dr0p05": 0.8181495666503906, "metrics/particle/matching/target_coverage_dr0p10": 0.8663105964660645, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7465896010398865, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8702042102813721, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014966697432100773, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07817871123552322, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07049228996038437, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01534027885645628, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12530511617660522, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030984622426331043, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3051416277885437, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0010016723535954952, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2391654253005981, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8714.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2633201479911804, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8689444661140442, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8229556679725647, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7193176746368408, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11395326256752014, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.946231842041016, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01259736530482769, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6743157505989075, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.042609281837940216, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.530984401702881, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10399.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.810680091381073, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.20499977469444275, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.170061111450195, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.368767738342285, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.19360703229904175, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.34540024399757385, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.406888008117676, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.045368604362010956, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.18179988861084, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023538770154118538, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08619019389152527, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08163649588823318, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019643738865852356, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13454167544841766, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.018917499110102654, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8822436332702637, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.06865861266851425, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.057272434234619, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1903.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6852092146873474, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4160925149917603, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.50276517868042, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.356405019760132, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2384669929742813, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 20.174243927001953, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06671398133039474, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.541764259338379, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.11881622672080994, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.053248405456543, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2818.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6194661259651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7474192380905151, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.694193363189697, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9174126386642456, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04138656333088875, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.17753605544567108, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.559374809265137, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3740474581718445, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.989995956420898, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010999863967299461, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08412298560142517, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07119274139404297, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016791053116321564, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12766294181346893, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006756405346095562, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10342393815517426, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.012564784847199917, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5961061120033264, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19241.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13150110840797424, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6296624541282654, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.42890769243240356, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3372425138950348, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08111628144979477, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.080280303955078, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0022325580939650536, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21968963742256165, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006714182440191507, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.4429261684417725, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21500.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8764234185218811, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.3708750903606415, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.008472442626953, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.097969055175781, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.1363111138343811, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.17853054404258728, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.463104724884033, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.11641768366098404, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 5.764926433563232, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012956744059920311, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08251990377902985, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07165394723415375, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01654946431517601, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12743118405342102, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025453814305365086, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2119329571723938, "metrics/particle/momentum/all/accepted/pz_bias": -0.004013338126242161, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1282092332839966, "metrics/particle/momentum/all/accepted/samples": 29858.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2159304916858673, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7651711106300354, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7152756452560425, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6155844926834106, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1037243977189064, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.24217414855957, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010571190156042576, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.544350802898407, "metrics/particle/momentum/all/assigned/pz_bias": 0.01824936456978321, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9107539653778076, "metrics/particle/momentum/all/assigned/samples": 34717.0, "metrics/particle/momentum/all/efficiency": 0.8346051573753357, "metrics/particle/momentum/all/event/energy_bias": -0.5815439224243164, "metrics/particle/momentum/all/event/p3_error_mean": 11.328712463378906, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.455883026123047, "metrics/particle/momentum/all/event/px_bias": 0.01590932160615921, "metrics/particle/momentum/all/event/py_bias": 0.7014667987823486, "metrics/particle/momentum/all/event/pz_abs_mean": 8.66252613067627, "metrics/particle/momentum/all/event/pz_bias": 0.21226118505001068, "metrics/particle/momentum/all/event/pz_residual_rms": 16.593164443969727, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00658694701269269, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03808354213833809, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.019080787897109985, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.002817461034283042, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05698557570576668, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013399927411228418, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10441140830516815, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.00240120617672801, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8670579195022583, "metrics/particle/momentum/class_1/accepted/samples": 16418.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03359554335474968, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09604605287313461, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07977933436632156, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.053420405834913254, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016595542430877686, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.830977201461792, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0037932670675218105, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2025631219148636, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.015155264176428318, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.179929494857788, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9730338454246521, "metrics/particle/momentum/class_1/event/energy_bias": 1.3956170082092285, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.160898208618164, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.103466033935547, "metrics/particle/momentum/class_1/event/px_bias": -0.3314160704612732, "metrics/particle/momentum/class_1/event/py_bias": -0.2711864709854126, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.256332874298096, "metrics/particle/momentum/class_1/event/pz_bias": 0.5268406271934509, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.822341918945312, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03434417396783829, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1910586804151535, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1922764629125595, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.022841528058052063, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23299705982208252, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03143712505698204, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.613497018814087, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.0005665982025675476, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.758129119873047, "metrics/particle/momentum/class_2/accepted/samples": 1336.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8456294536590576, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6328896284103394, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7045630812644958, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.24609249830245972, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33714285492897034, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.4507668018341064, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06857142597436905, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.895606279373169, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.05746423825621605, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.068934917449951, "metrics/particle/momentum/class_2/assigned/samples": 2625.0, "metrics/particle/momentum/class_2/efficiency": 0.4534962773323059, "metrics/particle/momentum/class_2/event/energy_bias": -7.194080829620361, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.3902587890625, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.447154521942139, "metrics/particle/momentum/class_2/event/px_bias": 0.34229201078414917, "metrics/particle/momentum/class_2/event/py_bias": 0.41859138011932373, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.036104202270508, "metrics/particle/momentum/class_2/event/pz_bias": -0.4164034128189087, "metrics/particle/momentum/class_2/event/pz_residual_rms": 15.016289710998535, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.01970829628407955, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13492457568645477, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13370616734027863, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04107458516955376, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17510241270065308, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010359116131439805, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2035907655954361, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.012015205807983875, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7510249614715576, "metrics/particle/momentum/class_3/accepted/samples": 11584.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3176650106906891, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5468791723251343, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4375057220458984, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3156696557998657, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16462251543998718, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.107780456542969, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008244752883911133, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5231704115867615, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.01980345882475376, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3196582794189453, "metrics/particle/momentum/class_3/assigned/samples": 14676.0, "metrics/particle/momentum/class_3/efficiency": 0.7516221404075623, "metrics/particle/momentum/class_3/event/energy_bias": 5.679408550262451, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.533891677856445, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.554052352905273, "metrics/particle/momentum/class_3/event/px_bias": -0.25967130064964294, "metrics/particle/momentum/class_3/event/py_bias": 0.44458070397377014, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.263519763946533, "metrics/particle/momentum/class_3/event/pz_bias": 0.34367379546165466, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.877418518066406, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010902398265898228, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05136832967400551, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05186305567622185, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.005171554628759623, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10051226615905762, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22144240140914917, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.0227106511592865, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7820203900337219, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09030020982027054, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5860543251037598, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4744200706481934, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4238274097442627, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.04696132615208626, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.285228729248047, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2764289081096649, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.05013011768460274, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9065901637077332, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.2926770746707916, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3835361003875732, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.879360556602478, "metrics/particle/momentum/class_4/event/px_bias": 0.205012708902359, "metrics/particle/momentum/class_4/event/py_bias": -0.00782107189297676, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9060957431793213, "metrics/particle/momentum/class_4/event/pz_bias": -0.024889061227440834, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.3196845054626465, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004484379664063454, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015728354454040527, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014986621215939522, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.00458332197740674, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04580669477581978, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1338137537240982, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06509819626808167, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.528461754322052, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0830639898777008, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03555694967508316, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03371284157037735, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0013607273576781154, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1322823315858841, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.557697057723999, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.24982690811157227, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.4008402824401855, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.16981180012226105, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8251095414161682, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5474869608879089, "metrics/particle/momentum/class_5/event/px_bias": 0.05969197303056717, "metrics/particle/momentum/class_5/event/py_bias": 0.1173022985458374, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5103901028633118, "metrics/particle/momentum/class_5/event/pz_bias": -0.21696078777313232, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2923924922943115, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6645739078521729, "step_valid_time": 16049.857724905014, "step_total_time": 16051.522299051285, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0017077185997043, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.055711568925614685, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2890, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9542009884678748, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573553719008264, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045799011532125206, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9585406301824212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522240527182867, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9553719008264463, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.960530679933665, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045569683795113, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06468798717301472, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2141, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529516200621394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9550711743772242, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.954010219951122, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0449288256227758, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502885042166, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9524021352313167, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95134414574539, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529516200621394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0055459929952264, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0613411305464544, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377431906614786, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362859362859363, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9370139968895801, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06371406371406371, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9361867704280156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9347319347319347, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354587869362364, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377431906614786} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_38000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_38000.json new file mode 100644 index 0000000000000000000000000000000000000000..c8bd598def0f11f7ae53ef881107f88d4be2475f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_38000.json @@ -0,0 +1 @@ +{"epoch": 1.8002653022550692, "train": {"Classification_binary": 2.2575485706329346, "Classification": 0.02009933441877365, "Regression_pt": 0.12794999778270721, "Regression_eta": 0.007338181138038635, "Regression_sin_phi": 0.007687278091907501, "Regression_cos_phi": 0.007461674511432648, "Regression_energy": 0.12855425477027893, "Total": 8.099444389343262, "Total_unweighted": 2.556639291346073}, "train_group": {}, "valid": {"Classification_binary": 2.348117192586263, "Classification": 0.018513298283020656, "Regression_pt": 0.11899189154307048, "Regression_eta": 0.009982677176594734, "Regression_sin_phi": 0.0077755941698948545, "Regression_cos_phi": 0.007860605294505755, "Regression_energy": 0.12241148948669434, "Total": 8.065446853637695, "Total_unweighted": 2.633652748540044, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018993332982063293, "diagnostic/loss/clic_pf/Classification_binary": 2.3571863174438477, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007328823674470186, "diagnostic/loss/clic_pf/Regression_energy": 0.12520694732666016, "diagnostic/loss/clic_pf/Regression_eta": 0.00913938693702221, "diagnostic/loss/clic_pf/Regression_pt": 0.12200260162353516, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007354323752224445, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03199716657400131, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18293872475624084, "diagnostic/regression/clic_pf/energy_residual_mean": 0.047550786286592484, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41461408138275146, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0010857143206521869, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1757601797580719, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06775882095098495, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4162437915802002, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9729660153388977, "metrics/particle/class_1/pid_accuracy": 0.9876192808151245, "metrics/particle/class_2/efficiency": 0.46199819445610046, "metrics/particle/class_2/pid_accuracy": 0.7917988896369934, "metrics/particle/class_3/efficiency": 0.7597370147705078, "metrics/particle/class_3/pid_accuracy": 0.9867226481437683, "metrics/particle/class_4/efficiency": 0.9457364082336426, "metrics/particle/class_4/pid_accuracy": 0.5420082211494446, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7682926654815674, "metrics/particle/count/bias_mean": -0.5183006525039673, "metrics/particle/count/mae": 3.004575252532959, "metrics/particle/count/prediction_mean": 69.48104858398438, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04336521774530411, "metrics/particle/event/energy_response_mean": 1.0060864686965942, "metrics/particle/event/met_abs_error": 3.830129861831665, "metrics/particle/event/scalar_pt_relative_abs_error": 0.033522285521030426, "metrics/particle/event/scalar_pt_response_mean": 1.0077440738677979, "metrics/particle/event/vector_pt_closure": 0.034385066479444504, "metrics/particle/matched/delta_eta_abs_mean": 0.007745107635855675, "metrics/particle/matched/delta_phi_abs_mean": 0.008579971268773079, "metrics/particle/matched/delta_r_mean": 0.013026946224272251, "metrics/particle/matched/energy_relative_abs_mean": 0.08263590931892395, "metrics/particle/matched/pid_accuracy": 0.9722816348075867, "metrics/particle/matched/pt_relative_abs_mean": 0.07240600138902664, "metrics/particle/matching/duplicate_fraction": 0.025313716381788254, "metrics/particle/matching/efficiency": 0.8381030559539795, "metrics/particle/matching/f1": 0.8412173986434937, "metrics/particle/matching/purity": 0.8443549871444702, "metrics/particle/matching/target_coverage_dr0p05": 0.8216976523399353, "metrics/particle/matching/target_coverage_dr0p10": 0.8700827956199646, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7483356595039368, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8729960322380066, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015051755122840405, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07798110693693161, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07112143188714981, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014805898070335388, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12605898082256317, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030846567824482918, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3026506006717682, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.005464933812618256, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1568100452423096, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8753.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26125428080558777, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7881194353103638, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8021947145462036, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6953151822090149, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11681483685970306, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.4250006675720215, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012140330858528614, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.680422306060791, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.05225960165262222, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7625718116760254, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10461.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8143082857131958, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.3315469026565552, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.447679042816162, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.3480589389801025, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3588794469833374, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.08025621622800827, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.749183654785156, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.3942323923110962, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 9.00521183013916, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023914847522974014, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0876772552728653, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0861654132604599, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02434348873794079, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14137108623981476, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020280811935663223, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8797224760055542, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.13295219838619232, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.965756416320801, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1923.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.671362042427063, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0763241052627563, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.699068546295166, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.5585696697235107, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22464022040367126, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 46.954978942871094, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06212706118822098, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.461016893386841, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.013335159048438072, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.953550338745117, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2849.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6259765625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.2422821521759033, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.371326446533203, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0159679651260376, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.019965969026088715, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.11487146466970444, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.232708930969238, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.01484202966094017, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.516172409057617, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011145095340907574, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08426598459482193, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07173723727464676, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016879653558135033, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12851198017597198, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.000621600600425154, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10089375078678131, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 4.3152958824066445e-05, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5726450681686401, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19305.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13104212284088135, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5278192758560181, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.39485299587249756, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3019348680973053, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08155501633882523, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.843130111694336, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002319539897143841, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.20965835452079773, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0007990189478732646, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1019392013549805, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21556.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8793386220932007, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.042656321078538895, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.675457000732422, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 3.8883466720581055, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.12780626118183136, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.30070018768310547, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.238816976547241, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.045168884098529816, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.149814605712891, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013104710727930069, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08264990895986557, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0724828839302063, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016752952709794044, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1286686211824417, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.00260164774954319, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20975154638290405, "metrics/particle/momentum/all/accepted/pz_bias": 0.006959928199648857, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.079817771911621, "metrics/particle/momentum/all/accepted/samples": 29981.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21426132321357727, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6507381796836853, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7053535580635071, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6043586134910583, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10382607579231262, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.53274917602539, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010153157636523247, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5348686575889587, "metrics/particle/momentum/all/assigned/pz_bias": 0.015084023587405682, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8652396202087402, "metrics/particle/momentum/all/assigned/samples": 34866.0, "metrics/particle/momentum/all/efficiency": 0.8380433320999146, "metrics/particle/momentum/all/event/energy_bias": -0.8680789470672607, "metrics/particle/momentum/all/event/p3_error_mean": 11.431639671325684, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.196926593780518, "metrics/particle/momentum/all/event/px_bias": 0.46671971678733826, "metrics/particle/momentum/all/event/py_bias": 0.495827853679657, "metrics/particle/momentum/all/event/pz_abs_mean": 8.955721855163574, "metrics/particle/momentum/all/event/pz_bias": 0.42455926537513733, "metrics/particle/momentum/all/event/pz_residual_rms": 17.189313888549805, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0063920821994543076, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03771389275789261, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.019318921491503716, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0033808539155870676, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05742168053984642, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001280253636650741, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10144640505313873, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005613231565803289, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7822967767715454, "metrics/particle/momentum/class_1/accepted/samples": 16403.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03559994697570801, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10153338313102722, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08545506745576859, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05912920832633972, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017069701105356216, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7629475593566895, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0034969178959727287, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.19304080307483673, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.018753550946712494, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.053952693939209, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9721448421478271, "metrics/particle/momentum/class_1/event/energy_bias": 1.2505593299865723, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.8369975090026855, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.057760238647461, "metrics/particle/momentum/class_1/event/px_bias": -0.32801544666290283, "metrics/particle/momentum/class_1/event/py_bias": -0.15867841243743896, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.87510871887207, "metrics/particle/momentum/class_1/event/pz_bias": 0.33283185958862305, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.454755783081055, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03597715124487877, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18929964303970337, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1899968832731247, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026271458715200424, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23006072640419006, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030080704018473625, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5668777227401733, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03125064820051193, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.616130828857422, "metrics/particle/momentum/class_2/accepted/samples": 1363.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8536100387573242, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6884450316429138, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7839735150337219, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.3304067850112915, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3370744585990906, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.6031107902526855, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06958473473787308, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9679975509643555, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.11608175188302994, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.212067127227783, "metrics/particle/momentum/class_2/assigned/samples": 2673.0, "metrics/particle/momentum/class_2/efficiency": 0.4626612365245819, "metrics/particle/momentum/class_2/event/energy_bias": -6.600865840911865, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.110599517822266, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.308964252471924, "metrics/particle/momentum/class_2/event/px_bias": 0.7500520944595337, "metrics/particle/momentum/class_2/event/py_bias": 0.0847172960639, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.928154945373535, "metrics/particle/momentum/class_2/event/pz_bias": -0.383740097284317, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.732867240905762, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020076429471373558, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1352328658103943, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13488303124904633, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04103443771600723, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17715691030025482, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0013678721152245998, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20456530153751373, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.007077791262418032, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7666588425636292, "metrics/particle/momentum/class_3/accepted/samples": 11697.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3080804646015167, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.2538938522338867, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3877217769622803, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.263016700744629, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1630912870168686, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.91399574279785, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007240982726216316, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49065330624580383, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0385693684220314, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2109577655792236, "metrics/particle/momentum/class_3/assigned/samples": 14777.0, "metrics/particle/momentum/class_3/efficiency": 0.7589540481567383, "metrics/particle/momentum/class_3/event/energy_bias": 5.016653537750244, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.364934921264648, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.463291168212891, "metrics/particle/momentum/class_3/event/px_bias": -0.16666947305202484, "metrics/particle/momentum/class_3/event/py_bias": 0.55604088306427, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.163305282592773, "metrics/particle/momentum/class_3/event/pz_bias": 0.5727940797805786, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.107962608337402, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.009779064916074276, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.049210209399461746, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.049733661115169525, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.005674760323017836, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.08972766995429993, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2130167931318283, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.014732079580426216, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.71685791015625, "metrics/particle/momentum/class_4/accepted/samples": 341.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07642800360918045, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.6401435136795044, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4858410358428955, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.437294840812683, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.04696132615208626, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 22.31690788269043, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.28838855028152466, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0482131689786911, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9199962615966797, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9419889450073242, "metrics/particle/momentum/class_4/event/energy_bias": -0.3545847237110138, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.4354965686798096, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9275938272476196, "metrics/particle/momentum/class_4/event/px_bias": 0.15155461430549622, "metrics/particle/momentum/class_4/event/py_bias": -0.016903547570109367, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9103919267654419, "metrics/particle/momentum/class_4/event/pz_bias": 0.04996649920940399, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.9396731853485107, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004731527529656887, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015203473158180714, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01451500691473484, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003995762672275305, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.039593592286109924, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1324414759874344, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.021289020776748657, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5105049014091492, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04352327063679695, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.07049747556447983, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0618620403110981, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03768979758024216, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.45010286569595337, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5687206387519836, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.17961686849594116, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.374722957611084, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.17984111607074738, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8422924280166626, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5270193219184875, "metrics/particle/momentum/class_5/event/px_bias": 0.0597979873418808, "metrics/particle/momentum/class_5/event/py_bias": 0.03065158985555172, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.564083456993103, "metrics/particle/momentum/class_5/event/pz_bias": -0.1472930759191513, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.4136393070220947, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.837024450302124, "step_valid_time": 16484.158290863037, "step_total_time": 16485.995315790176, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006252215740874, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05440042866031236, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2890, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9585406301824212, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9544253632760898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564785702465662, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04557463672391017, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9572139303482587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9531043593130779, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9551547244745987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9585406301824212, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015586522988422, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06449632997039145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2141, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2137, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9502885042166, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9540998217468806, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9521903491216367, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04590017825311943, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9485130936529073, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9523172905525846, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9504113853680232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9502885042166, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003961539624709, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06248453586967917, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1194, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9321372854914196, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9310479158550837, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06786271450858035, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9291828793774319, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9313572542901716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.930268796260226, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299610894941635} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_39000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_39000.json new file mode 100644 index 0000000000000000000000000000000000000000..cb792a3d891a5258682c3af40cd9047664c49537 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_39000.json @@ -0,0 +1 @@ +{"epoch": 1.847640704945992, "train": {"Classification_binary": 2.266429901123047, "Classification": 0.019133873283863068, "Regression_pt": 0.11718007922172546, "Regression_eta": 0.009045851416885853, "Regression_sin_phi": 0.007248355075716972, "Regression_cos_phi": 0.006545541808009148, "Regression_energy": 0.11695663630962372, "Total": 7.743195056915283, "Total_unweighted": 2.542540238238871}, "train_group": {}, "valid": {"Classification_binary": 2.3578133583068848, "Classification": 0.018728567908207577, "Regression_pt": 0.11959682901700337, "Regression_eta": 0.009985854228337606, "Regression_sin_phi": 0.007861350352565447, "Regression_cos_phi": 0.007833654060959816, "Regression_energy": 0.12302021185557048, "Total": 8.104931513468424, "Total_unweighted": 2.644839825729529, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.019124748185276985, "diagnostic/loss/clic_pf/Classification_binary": 2.363567352294922, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00733815785497427, "diagnostic/loss/clic_pf/Regression_energy": 0.12617427110671997, "diagnostic/loss/clic_pf/Regression_eta": 0.009097518399357796, "diagnostic/loss/clic_pf/Regression_pt": 0.1228497326374054, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.00736822048202157, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02269795909523964, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18203547596931458, "diagnostic/regression/clic_pf/energy_residual_mean": 0.03825158253312111, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4132159948348999, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.010282892733812332, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17478041350841522, "diagnostic/regression/clic_pf/pt_residual_mean": 0.05856164172291756, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41446399688720703, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9739996194839478, "metrics/particle/class_1/pid_accuracy": 0.9879589676856995, "metrics/particle/class_2/efficiency": 0.4677492082118988, "metrics/particle/class_2/pid_accuracy": 0.775795578956604, "metrics/particle/class_3/efficiency": 0.7625606060028076, "metrics/particle/class_3/pid_accuracy": 0.987987220287323, "metrics/particle/class_4/efficiency": 0.9505813717842102, "metrics/particle/class_4/pid_accuracy": 0.5616717338562012, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7510204315185547, "metrics/particle/count/bias_mean": -0.7156862616539001, "metrics/particle/count/mae": 3.0411765575408936, "metrics/particle/count/prediction_mean": 69.28366088867188, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04397949203848839, "metrics/particle/event/energy_response_mean": 1.0023854970932007, "metrics/particle/event/met_abs_error": 4.033290386199951, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03284153714776039, "metrics/particle/event/scalar_pt_response_mean": 1.0036754608154297, "metrics/particle/event/vector_pt_closure": 0.0362432524561882, "metrics/particle/matched/delta_eta_abs_mean": 0.0075119854882359505, "metrics/particle/matched/delta_phi_abs_mean": 0.009019233286380768, "metrics/particle/matched/delta_r_mean": 0.013187626376748085, "metrics/particle/matched/energy_relative_abs_mean": 0.08224418759346008, "metrics/particle/matched/pid_accuracy": 0.9722546339035034, "metrics/particle/matched/pt_relative_abs_mean": 0.07203276455402374, "metrics/particle/matching/duplicate_fraction": 0.02494245581328869, "metrics/particle/matching/efficiency": 0.8403159976005554, "metrics/particle/matching/f1": 0.8446338176727295, "metrics/particle/matching/purity": 0.8489962816238403, "metrics/particle/matching/target_coverage_dr0p05": 0.8210721015930176, "metrics/particle/matching/target_coverage_dr0p10": 0.8688690066337585, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7512208223342896, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8747233748435974, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015040850266814232, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07824259251356125, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07111791521310806, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.010805928148329258, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1260245144367218, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002968714339658618, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3052191138267517, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0027870561461895704, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1731619834899902, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8758.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26398372650146484, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9663048982620239, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8783053159713745, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.767769992351532, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11370821297168732, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.093635559082031, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011399559676647186, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6576573848724365, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.0030206076335161924, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.573728322982788, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10439.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.814773440361023, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.035018883645534515, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.43428373336792, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.4350569248199463, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.18434780836105347, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.29921475052833557, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.666232109069824, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.5803601145744324, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.833508491516113, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02352169156074524, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08299397677183151, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08129645138978958, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.018531398847699165, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13490088284015656, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02156759612262249, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8949305415153503, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09993696957826614, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.056960105895996, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1901.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6725313067436218, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0952256917953491, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 22482.861328125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 22482.7109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23523207008838654, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1198884.0, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06610407680273056, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.449793577194214, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.21051886677742004, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.706279754638672, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2844.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6188151240348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.6436198949813843, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.836889266967773, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9211068153381348, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0863913893699646, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12969215214252472, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.695354461669922, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.13229471445083618, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.146965026855469, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011456473730504513, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08389153331518173, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07158017158508301, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.007482788059860468, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12733404338359833, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00046379800187423825, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.09929242730140686, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007118844427168369, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5463040471076965, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19405.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12966036796569824, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5070157051086426, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4025198817253113, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.30387601256370544, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07689453661441803, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.59060525894165, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0020870049484074116, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2160237580537796, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0017496212385594845, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.6574872732162476, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21562.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.883893609046936, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.18909025192260742, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.400167465209961, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.4174394607543945, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2400486022233963, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.12332839518785477, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.708127021789551, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.1435692012310028, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 9.447937965393066, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013263550586998463, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0821891725063324, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07205989211797714, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.009149481542408466, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12744715809822083, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002527940319851041, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20959091186523438, "metrics/particle/momentum/all/accepted/pz_bias": 0.0009123862255364656, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.088334321975708, "metrics/particle/momentum/all/accepted/samples": 30064.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21420983970165253, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6926202178001404, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1835.5318603515625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1835.4251708984375, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10084660351276398, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 342508.8125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010101879946887493, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5306472182273865, "metrics/particle/momentum/all/assigned/pz_bias": 0.015194667503237724, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9208784103393555, "metrics/particle/momentum/all/assigned/samples": 34845.0, "metrics/particle/momentum/all/efficiency": 0.8403633832931519, "metrics/particle/momentum/all/event/energy_bias": -1.8677289485931396, "metrics/particle/momentum/all/event/p3_error_mean": 11.818657875061035, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.6495537757873535, "metrics/particle/momentum/all/event/px_bias": -0.14209213852882385, "metrics/particle/momentum/all/event/py_bias": 0.5522353053092957, "metrics/particle/momentum/all/event/pz_abs_mean": 9.114954948425293, "metrics/particle/momentum/all/event/pz_bias": -0.5916345715522766, "metrics/particle/momentum/all/event/pz_residual_rms": 18.368146896362305, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006673740688711405, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03759051486849785, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01915399543941021, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0029279948212206364, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05624118819832802, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012792397756129503, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.09927671402692795, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.00026710270321927965, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8032877445220947, "metrics/particle/momentum/class_1/accepted/samples": 16416.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03562003746628761, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.0965471863746643, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.0843638926744461, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05800455063581467, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016062114387750626, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9344054460525513, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003733997233211994, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.20075850188732147, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.022658633068203926, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.4661316871643066, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9729152917861938, "metrics/particle/momentum/class_1/event/energy_bias": 1.1638668775558472, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.07323169708252, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.174493312835693, "metrics/particle/momentum/class_1/event/px_bias": -0.6175339818000793, "metrics/particle/momentum/class_1/event/py_bias": -0.24802373349666595, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.110161781311035, "metrics/particle/momentum/class_1/event/pz_bias": 0.3336545526981354, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.571690559387207, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03583122044801712, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19382627308368683, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19509606063365936, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.033167097717523575, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23460496962070465, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02949640341103077, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5837509632110596, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.02884570322930813, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5894131660461426, "metrics/particle/momentum/class_2/accepted/samples": 1390.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8188790678977966, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.635123610496521, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 23973.546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 23973.08203125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3295837938785553, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1238028.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06636670231819153, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8219306468963623, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.068443164229393, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.737340927124023, "metrics/particle/momentum/class_2/assigned/samples": 2667.0, "metrics/particle/momentum/class_2/efficiency": 0.4718261957168579, "metrics/particle/momentum/class_2/event/energy_bias": -7.404101848602295, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.043492317199707, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.2409796714782715, "metrics/particle/momentum/class_2/event/px_bias": 0.19751989841461182, "metrics/particle/momentum/class_2/event/py_bias": -0.21453410387039185, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.884812355041504, "metrics/particle/momentum/class_2/event/pz_bias": -0.09316345304250717, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.429224967956543, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.02001050114631653, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13325482606887817, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13292734324932098, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.022960899397730827, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17424121499061584, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011928090825676918, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20170842111110687, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0010025266092270613, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7632133960723877, "metrics/particle/momentum/class_3/accepted/samples": 11737.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.31449201703071594, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3744922876358032, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3380004167556763, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.202049732208252, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15905703604221344, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 11.75354290008545, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007383823394775391, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4997342824935913, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.003560252720490098, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1766135692596436, "metrics/particle/momentum/class_3/assigned/samples": 14762.0, "metrics/particle/momentum/class_3/efficiency": 0.7615494132041931, "metrics/particle/momentum/class_3/event/energy_bias": 5.100428581237793, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.450645446777344, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.575644016265869, "metrics/particle/momentum/class_3/event/px_bias": 0.009099118411540985, "metrics/particle/momentum/class_3/event/py_bias": 0.9019575715065002, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.149190902709961, "metrics/particle/momentum/class_3/event/pz_bias": -0.6919063925743103, "metrics/particle/momentum/class_3/event/pz_residual_rms": 10.786091804504395, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011061804369091988, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05260622501373291, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05328308418393135, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.008784628473222256, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09824663400650024, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22829948365688324, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.022776028141379356, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.771513819694519, "metrics/particle/momentum/class_4/accepted/samples": 345.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07640223950147629, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.423518180847168, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3181970119476318, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.268187165260315, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.033149171620607376, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.09121322631836, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2909117639064789, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.013658326119184494, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.0175561904907227, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9530386924743652, "metrics/particle/momentum/class_4/event/energy_bias": -0.3737201988697052, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.374324083328247, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8895975351333618, "metrics/particle/momentum/class_4/event/px_bias": 0.18962588906288147, "metrics/particle/momentum/class_4/event/py_bias": 0.07506997138261795, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8706445693969727, "metrics/particle/momentum/class_4/event/pz_bias": 0.05004953220486641, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.7309420108795166, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0040584453381598, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.012896687723696232, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.012739252299070358, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003315757028758526, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.03596162050962448, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13514679670333862, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06925969570875168, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6146292686462402, "metrics/particle/momentum/class_5/accepted/samples": 176.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.049569129943847656, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03276294469833374, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03197071701288223, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.012113242410123348, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.021978022530674934, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12562328577041626, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.520487368106842, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4559794068336487, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.256309509277344, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9670329689979553, "metrics/particle/momentum/class_5/event/energy_bias": -0.35420262813568115, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8214608430862427, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5335866212844849, "metrics/particle/momentum/class_5/event/px_bias": 0.07919688522815704, "metrics/particle/momentum/class_5/event/py_bias": 0.03776556998491287, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5283677577972412, "metrics/particle/momentum/class_5/event/pz_bias": -0.190268874168396, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.3130598068237305, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5759837627410889, "step_valid_time": 16919.04123401642, "step_total_time": 16920.617218255997, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979970243143443, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.056430987564174306, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2892, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2885, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9592039800995025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956982131039047, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.958091767434156, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04301786896095301, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.956882255389718, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9546657842488419, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.955772734802054, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9592039800995025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999478895144506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06280034429877046, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2141, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9554367201426025, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535245719368468, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.044563279857397504, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502885042166, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540998217468806, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9521903491216367, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516200621393698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0000276948929216, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06598243511259239, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1279, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1194, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9291828793774319, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9335418295543393, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9313572542901716, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06645817044566067, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9268482490272374, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9311962470680218, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9290171606864275, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9291828793774319} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_4000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_4000.json new file mode 100644 index 0000000000000000000000000000000000000000..d77f212ca1b903e8ee1fab550ca2b22c2c199040 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_4000.json @@ -0,0 +1 @@ +{"epoch": 0.18950161076369149, "train": {"Classification_binary": 3.227652072906494, "Classification": 0.030978890135884285, "Regression_pt": 0.16820381581783295, "Regression_eta": 0.014492662623524666, "Regression_sin_phi": 0.011179630644619465, "Regression_cos_phi": 0.011586465872824192, "Regression_energy": 0.17254982888698578, "Total": 11.565425872802734, "Total_unweighted": 3.6366433668881655}, "train_group": {}, "valid": {"Classification_binary": 3.224571863810221, "Classification": 0.028996023039023083, "Regression_pt": 0.1676547328631083, "Regression_eta": 0.012399649868408838, "Regression_sin_phi": 0.009545854603250822, "Regression_cos_phi": 0.009651665265361467, "Regression_energy": 0.17068594694137573, "Total": 11.120051066080729, "Total_unweighted": 3.6235057363907495, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.029540132731199265, "diagnostic/loss/clic_pf/Classification_binary": 3.201523542404175, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009266047738492489, "diagnostic/loss/clic_pf/Regression_energy": 0.17307370901107788, "diagnostic/loss/clic_pf/Regression_eta": 0.011609922163188457, "diagnostic/loss/clic_pf/Regression_pt": 0.17079824209213257, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009226178750395775, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04962446168065071, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.21652337908744812, "diagnostic/regression/clic_pf/energy_residual_mean": 0.06517808139324188, "diagnostic/regression/clic_pf/energy_residual_rms": 0.48685115575790405, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.030442776158452034, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.214400514960289, "diagnostic/regression/clic_pf/pt_residual_mean": 0.09928731620311737, "diagnostic/regression/clic_pf/pt_residual_rms": 0.49469879269599915, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.957063615322113, "metrics/particle/class_1/pid_accuracy": 0.985378086566925, "metrics/particle/class_2/efficiency": 0.367275595664978, "metrics/particle/class_2/pid_accuracy": 0.5551120638847351, "metrics/particle/class_3/efficiency": 0.6966052651405334, "metrics/particle/class_3/pid_accuracy": 0.9790525436401367, "metrics/particle/class_4/efficiency": 0.9292635917663574, "metrics/particle/class_4/pid_accuracy": 0.2679874897003174, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.546012282371521, "metrics/particle/count/bias_mean": 0.8705882430076599, "metrics/particle/count/mae": 3.8143789768218994, "metrics/particle/count/prediction_mean": 70.86993408203125, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.07257422059774399, "metrics/particle/event/energy_response_mean": 1.0228697061538696, "metrics/particle/event/met_abs_error": 8.469635963439941, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06467630714178085, "metrics/particle/event/scalar_pt_response_mean": 1.039031982421875, "metrics/particle/event/vector_pt_closure": 0.0659610703587532, "metrics/particle/matched/delta_eta_abs_mean": 0.0151865528896451, "metrics/particle/matched/delta_phi_abs_mean": 0.013752229511737823, "metrics/particle/matched/delta_r_mean": 0.02286997251212597, "metrics/particle/matched/energy_relative_abs_mean": 0.08717985451221466, "metrics/particle/matched/pid_accuracy": 0.9559239745140076, "metrics/particle/matched/pt_relative_abs_mean": 0.07966797053813934, "metrics/particle/matching/duplicate_fraction": 0.03803340345621109, "metrics/particle/matching/efficiency": 0.7952548861503601, "metrics/particle/matching/f1": 0.7903400659561157, "metrics/particle/matching/purity": 0.7854856848716736, "metrics/particle/matching/target_coverage_dr0p05": 0.7352076172828674, "metrics/particle/matching/target_coverage_dr0p10": 0.8347323536872864, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7012390494346619, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8417445421218872, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02852368727326393, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08272303640842438, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0779818668961525, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.025178851559758186, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13543936610221863, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004772977437824011, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3426884710788727, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.005810861010104418, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2898781299591064, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8171.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3681252896785736, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.2383695840835571, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.5365437269210815, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.4162518978118896, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1603899896144867, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 25.20733642578125, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.02007264457643032, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.9546034932136536, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.004784688353538513, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.269381046295166, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10462.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7601637244224548, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.8470573425292969, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 9.015275001525879, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.082969665527344, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.5108290910720825, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.5254806280136108, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.676509857177734, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.5297669172286987, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 13.461555480957031, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.04035163298249245, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08951576799154282, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.09273598343133926, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.032289911061525345, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.15005329251289368, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.015980331227183342, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.894934356212616, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.09971436858177185, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8952345848083496, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1627.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.877988874912262, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5764113664627075, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.5583581924438477, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.3912594318389893, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.30370113253593445, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 25.163061141967773, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0857834666967392, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2261693477630615, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.12421397119760513, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 9.774658203125, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2891.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5296223759651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.6572675704956055, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 9.082477569580078, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.576392650604248, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.1986944079399109, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.039916060864925385, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.877296447753906, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.48857781291007996, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.600908279418945, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018829092383384705, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08827687054872513, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07866942137479782, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.022686298936605453, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1363680362701416, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007512744632549584, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11643509566783905, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.0028566811233758926, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6437592506408691, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18635.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1784108579158783, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.9520048499107361, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.7636345028877258, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.6623597145080566, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.1092328280210495, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 8.75222110748291, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0034178560599684715, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.33049723505973816, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.030770542100071907, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 5.3736491203308105, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21651.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8488202691078186, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 6.869649410247803, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 11.687287330627441, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 9.567850112915039, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -3.90631103515625, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -2.199139356613159, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 5.987419128417969, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.1184654235839844, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 34.32912826538086, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.02284666709601879, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08675171434879303, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07927674800157547, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.023952141404151917, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13692334294319153, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002778461668640375, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.22600266337394714, "metrics/particle/momentum/all/accepted/pz_bias": -0.002163700060918927, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.108786702156067, "metrics/particle/momentum/all/accepted/samples": 28433.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.292891263961792, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0891636610031128, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.2254594564437866, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.1130644083023071, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.14058393239974976, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 17.01723289489746, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.015198263339698315, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7561854124069214, "metrics/particle/momentum/all/assigned/pz_bias": 0.00734353344887495, "metrics/particle/momentum/all/assigned/pz_residual_rms": 5.585661888122559, "metrics/particle/momentum/all/assigned/samples": 35004.0, "metrics/particle/momentum/all/efficiency": 0.7947728633880615, "metrics/particle/momentum/all/event/energy_bias": 4.3653244972229, "metrics/particle/momentum/all/event/p3_error_mean": 20.988502502441406, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.481937408447266, "metrics/particle/momentum/all/event/px_bias": -4.6158342361450195, "metrics/particle/momentum/all/event/py_bias": -2.764536142349243, "metrics/particle/momentum/all/event/pz_abs_mean": 15.202857971191406, "metrics/particle/momentum/all/event/pz_bias": 0.07727622985839844, "metrics/particle/momentum/all/event/pz_residual_rms": 40.299808502197266, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.014182492159307003, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04147625342011452, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02511569671332836, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.010004072450101376, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06514503061771393, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001054852269589901, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11427290737628937, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0076706926338374615, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7119131684303284, "metrics/particle/momentum/class_1/accepted/samples": 16116.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.06084342300891876, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.1691441535949707, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15392139554023743, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.12596024572849274, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.02868488058447838, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 3.4744601249694824, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005215432960540056, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.329438716173172, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.04775598645210266, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 6.297801971435547, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9551354050636292, "metrics/particle/momentum/class_1/event/energy_bias": 6.946743965148926, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.074772834777832, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.159612655639648, "metrics/particle/momentum/class_1/event/px_bias": -2.742659091949463, "metrics/particle/momentum/class_1/event/py_bias": -0.8332064747810364, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.00046157836914, "metrics/particle/momentum/class_1/event/pz_bias": 1.7029962539672852, "metrics/particle/momentum/class_1/event/pz_residual_rms": 37.5491828918457, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.044145140796899796, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21557919681072235, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.21392124891281128, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.055592238903045654, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2543717920780182, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04481792822480202, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8892805576324463, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.004097576253116131, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.327023506164551, "metrics/particle/momentum/class_2/accepted/samples": 1071.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0848180055618286, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8729913830757141, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 1.092089295387268, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.535657525062561, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.4307463765144348, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 4.08055305480957, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09617526829242706, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.761197328567505, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09327949583530426, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.284602165222168, "metrics/particle/momentum/class_2/assigned/samples": 2693.0, "metrics/particle/momentum/class_2/efficiency": 0.3635437786579132, "metrics/particle/momentum/class_2/event/energy_bias": -10.98332691192627, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.529430389404297, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.42701530456543, "metrics/particle/momentum/class_2/event/px_bias": -1.7373073101043701, "metrics/particle/momentum/class_2/event/py_bias": -2.356095314025879, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.850412368774414, "metrics/particle/momentum/class_2/event/pz_bias": -1.2828327417373657, "metrics/particle/momentum/class_2/event/pz_residual_rms": 21.219398498535156, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03408016636967659, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14403748512268066, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14890262484550476, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05367278680205345, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.19090217351913452, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001304388279095292, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22696971893310547, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.01896074414253235, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7703576683998108, "metrics/particle/momentum/class_3/accepted/samples": 10733.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.42021411657333374, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.144726276397705, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.4547650814056396, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.324777364730835, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.21841010451316833, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.39992904663086, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012219686061143875, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7067359089851379, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.015240311622619629, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.5421955585479736, "metrics/particle/momentum/class_3/assigned/samples": 14894.0, "metrics/particle/momentum/class_3/efficiency": 0.6964054107666016, "metrics/particle/momentum/class_3/event/energy_bias": 10.24759578704834, "metrics/particle/momentum/class_3/event/p3_error_mean": 13.06045150756836, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 6.885846138000488, "metrics/particle/momentum/class_3/event/px_bias": -0.48939192295074463, "metrics/particle/momentum/class_3/event/py_bias": 0.35457471013069153, "metrics/particle/momentum/class_3/event/pz_abs_mean": 9.598071098327637, "metrics/particle/momentum/class_3/event/pz_bias": -0.3565591275691986, "metrics/particle/momentum/class_3/event/pz_residual_rms": 15.996097564697266, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.017412809655070305, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.04948614165186882, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.051677487790584564, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.008109102956950665, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09664811939001083, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2634444236755371, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.032792843878269196, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9708185791969299, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11051645874977112, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.67756986618042, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.1805496215820312, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.1285345554351807, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 27.77092742919922, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4190317690372467, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.01811256632208824, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3808977603912354, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.7276719808578491, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.464691638946533, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.5774348974227905, "metrics/particle/momentum/class_4/event/px_bias": 0.22174137830734253, "metrics/particle/momentum/class_4/event/py_bias": 0.0598883256316185, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.6299222707748413, "metrics/particle/momentum/class_4/event/pz_bias": 0.4882051944732666, "metrics/particle/momentum/class_4/event/pz_residual_rms": 6.248878002166748, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.012047642841935158, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.026883071288466454, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02664482593536377, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.003164601046591997, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.0783587172627449, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.20578472316265106, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.016888301819562912, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.534127414226532, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03112269565463066, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.040209271013736725, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03981992229819298, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.010501857846975327, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.13393673300743103, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5723941326141357, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.35157230496406555, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.1766157150268555, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -1.1180161237716675, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.7666678428649902, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.209084153175354, "metrics/particle/momentum/class_5/event/px_bias": 0.13178226351737976, "metrics/particle/momentum/class_5/event/py_bias": 0.010302528738975525, "metrics/particle/momentum/class_5/event/pz_abs_mean": 1.0193796157836914, "metrics/particle/momentum/class_5/event/pz_bias": -0.47453346848487854, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.5665507316589355, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4451804161071777, "step_valid_time": 1737.3199338912964, "step_total_time": 1738.7651147842407, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0184103706757839, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10096552633487765, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2832, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2808, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9393034825870646, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9371277299801456, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9382143448732815, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0628722700198544, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9313432835820895, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9291859695565851, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9302633758489316, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9393034825870646, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0147937165913496, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10635031748625656, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2074, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2060, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9205503772747448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9205503772747448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9205503772747448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07944962272525521, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9143364403018198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9143364403018198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9143364403018198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9205503772747448, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0194215421622397, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11133422864859122, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1289, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1147, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1133, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8926070038910506, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8898370830100854, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8912198912198912, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11016291698991466, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8817120622568093, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8789759503491078, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8803418803418803, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8926070038910506} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_40000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..17402aab58aa94aeef130a9a97dcef27c53be647 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 1.895016107636915, "train": {"Classification_binary": 2.3895974159240723, "Classification": 0.020705634728074074, "Regression_pt": 0.12718656659126282, "Regression_eta": 0.010103692300617695, "Regression_sin_phi": 0.007626672275364399, "Regression_cos_phi": 0.00838391575962305, "Regression_energy": 0.12666086852550507, "Total": 8.413192749023438, "Total_unweighted": 2.6902647661045194}, "train_group": {}, "valid": {"Classification_binary": 2.3422180811564126, "Classification": 0.018459605673948925, "Regression_pt": 0.11914792656898499, "Regression_eta": 0.00995170883834362, "Regression_sin_phi": 0.007815991217891375, "Regression_cos_phi": 0.007819859310984612, "Regression_energy": 0.12258639931678772, "Total": 8.060129801432291, "Total_unweighted": 2.627999572083354, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018874334171414375, "diagnostic/loss/clic_pf/Classification_binary": 2.348249673843384, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007366543635725975, "diagnostic/loss/clic_pf/Regression_energy": 0.1251874715089798, "diagnostic/loss/clic_pf/Regression_eta": 0.009125160984694958, "diagnostic/loss/clic_pf/Regression_pt": 0.12194633483886719, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007350916508585215, "diagnostic/regression/clic_pf/energy_pred_mean": 0.0279067549854517, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18176518380641937, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04346038028597832, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41090792417526245, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005508176051080227, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17430269718170166, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06333636492490768, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4119139611721039, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9733635187149048, "metrics/particle/class_1/pid_accuracy": 0.9881961345672607, "metrics/particle/class_2/efficiency": 0.45771312713623047, "metrics/particle/class_2/pid_accuracy": 0.7868933081626892, "metrics/particle/class_3/efficiency": 0.7610302567481995, "metrics/particle/class_3/pid_accuracy": 0.9867735505104065, "metrics/particle/class_4/efficiency": 0.9437984228134155, "metrics/particle/class_4/pid_accuracy": 0.5451745390892029, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.753564178943634, "metrics/particle/count/bias_mean": -0.828104555606842, "metrics/particle/count/mae": 3.043790817260742, "metrics/particle/count/prediction_mean": 69.1712417602539, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.044597480446100235, "metrics/particle/event/energy_response_mean": 1.0072922706604004, "metrics/particle/event/met_abs_error": 4.020866870880127, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03341073915362358, "metrics/particle/event/scalar_pt_response_mean": 1.0059716701507568, "metrics/particle/event/vector_pt_closure": 0.03520853444933891, "metrics/particle/matched/delta_eta_abs_mean": 0.007305696606636047, "metrics/particle/matched/delta_phi_abs_mean": 0.008672301657497883, "metrics/particle/matched/delta_r_mean": 0.012744915671646595, "metrics/particle/matched/energy_relative_abs_mean": 0.08213155716657639, "metrics/particle/matched/pid_accuracy": 0.9724496006965637, "metrics/particle/matched/pt_relative_abs_mean": 0.07199861854314804, "metrics/particle/matching/duplicate_fraction": 0.024982992559671402, "metrics/particle/matching/efficiency": 0.8384672403335571, "metrics/particle/matching/f1": 0.8434563279151917, "metrics/particle/matching/purity": 0.8485051989555359, "metrics/particle/matching/target_coverage_dr0p05": 0.8219217658042908, "metrics/particle/matching/target_coverage_dr0p10": 0.868794322013855, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7501283884048462, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.87383633852005, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01497829519212246, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07670003175735474, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07007020711898804, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.013815246522426605, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12430482357740402, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028574694879353046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.2996850311756134, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.009020715020596981, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.166101336479187, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8749.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25698724389076233, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8117284178733826, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7265993356704712, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6175463199615479, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11485451459884644, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.88649320602417, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011868299916386604, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6538824439048767, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03112088516354561, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.448976755142212, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10448.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.813936173915863, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.17205122113227844, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.209689140319824, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2651970386505127, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1699979305267334, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.15248218178749084, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.4976420402526855, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.22390243411064148, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.615421295166016, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023495566099882126, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0865325853228569, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08500756323337555, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023903025314211845, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13961374759674072, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02049395628273487, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8699060678482056, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.12950225174427032, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8657543659210205, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1903.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6738191246986389, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1701085567474365, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 22676.06640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 22675.927734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23168595135211945, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1211101.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0669470727443695, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4013102054595947, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.18827754259109497, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.882068634033203, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2853.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6194661259651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.1021871566772461, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.0654706954956055, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0077095031738281, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.11660274118185043, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1446203589439392, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.9172821044921875, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.507481575012207, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 15.060863494873047, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010752297937870026, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08387307077646255, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07145887613296509, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01291123777627945, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12770508229732513, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005179200088605285, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.0993805080652237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.010434248484671116, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5502150058746338, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19308.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13048772513866425, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6337934136390686, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4220517873764038, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.32718175649642944, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07934368401765823, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.029386043548584, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0026029562577605247, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2303609848022461, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.013206846080720425, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.0497453212738037, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21514.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8794752955436707, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.1222822442650795, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.2182416915893555, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.283438682556152, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.11850611865520477, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.1179623231291771, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.5823771953582764, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.13843010365962982, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.315926551818848, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012795811519026756, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08194731920957565, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.071913942694664, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013873404823243618, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12751689553260803, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024699599016457796, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20681625604629517, "metrics/particle/momentum/all/accepted/pz_bias": 0.004135531838983297, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0553845167160034, "metrics/particle/momentum/all/accepted/samples": 29960.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2129749208688736, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7311414480209351, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1858.724365234375, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1858.6217041015625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1024845615029335, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 346695.46875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010656326077878475, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5353637337684631, "metrics/particle/momentum/all/assigned/pz_bias": 0.03292945399880409, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.877049446105957, "metrics/particle/momentum/all/assigned/samples": 34815.0, "metrics/particle/momentum/all/efficiency": 0.8374563455581665, "metrics/particle/momentum/all/event/energy_bias": -0.39652061462402344, "metrics/particle/momentum/all/event/p3_error_mean": 11.769477844238281, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.579928398132324, "metrics/particle/momentum/all/event/px_bias": -0.06511090695858002, "metrics/particle/momentum/all/event/py_bias": 0.41506487131118774, "metrics/particle/momentum/all/event/pz_abs_mean": 9.055492401123047, "metrics/particle/momentum/all/event/pz_bias": 0.8698140382766724, "metrics/particle/momentum/all/event/pz_residual_rms": 18.548940658569336, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006340631749480963, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03734024986624718, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.018802404403686523, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0027503063902258873, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05567555874586105, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014641288435086608, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10150228440761566, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.003290921449661255, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8116693496704102, "metrics/particle/momentum/class_1/accepted/samples": 16392.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.034014780074357986, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.08376069366931915, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.0737682431936264, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.04680699482560158, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.018077287822961807, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.578736662864685, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0033783784601837397, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.1904880255460739, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.024485599249601364, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.151608467102051, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9714929461479187, "metrics/particle/momentum/class_1/event/energy_bias": 2.675997018814087, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.500604629516602, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.433254241943359, "metrics/particle/momentum/class_1/event/px_bias": -0.5985321402549744, "metrics/particle/momentum/class_1/event/py_bias": -0.2790285348892212, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.449981689453125, "metrics/particle/momentum/class_1/event/pz_bias": 0.7975960373878479, "metrics/particle/momentum/class_1/event/pz_residual_rms": 16.337522506713867, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.035131316632032394, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18874172866344452, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19051983952522278, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.029126273468136787, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22958019375801086, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.029126213863492012, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5626745223999023, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.06336908787488937, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.4674272537231445, "metrics/particle/momentum/class_2/accepted/samples": 1339.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8415940999984741, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6503406763076782, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 24420.935546875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 24420.4765625, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3401283621788025, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1256870.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07059267908334732, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9108033180236816, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.21824458241462708, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.953226089477539, "metrics/particle/momentum/class_2/assigned/samples": 2649.0, "metrics/particle/momentum/class_2/efficiency": 0.45451459288597107, "metrics/particle/momentum/class_2/event/energy_bias": -7.457948207855225, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.253341674804688, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.315040111541748, "metrics/particle/momentum/class_2/event/px_bias": 0.45662543177604675, "metrics/particle/momentum/class_2/event/py_bias": 0.20597921311855316, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.025063514709473, "metrics/particle/momentum/class_2/event/pz_bias": 0.24290986359119415, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.833571434020996, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019487883895635605, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1341657042503357, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13423661887645721, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03480057790875435, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1759687215089798, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000939368037506938, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20041026175022125, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0004091767768841237, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7307080030441284, "metrics/particle/momentum/class_3/accepted/samples": 11710.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3099352717399597, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4813138246536255, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.2709591388702393, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1435930728912354, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.159050852060318, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 9.792941093444824, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008474576286971569, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5102844834327698, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.013762596063315868, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3033573627471924, "metrics/particle/momentum/class_3/assigned/samples": 14750.0, "metrics/particle/momentum/class_3/efficiency": 0.7597975730895996, "metrics/particle/momentum/class_3/event/energy_bias": 5.046247482299805, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.418871879577637, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.442671298980713, "metrics/particle/momentum/class_3/event/px_bias": -0.13050994277000427, "metrics/particle/momentum/class_3/event/py_bias": 0.4476572573184967, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.202334403991699, "metrics/particle/momentum/class_3/event/pz_bias": -0.0820515975356102, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.924830436706543, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010162651538848877, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.04908965900540352, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04966290295124054, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0027779231313616037, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09344927221536636, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.21547648310661316, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.01364993117749691, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.750909686088562, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06860703229904175, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.279725432395935, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.1869796514511108, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.141064167022705, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03591160103678703, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.171062469482422, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.24810796976089478, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02410207875072956, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.7950738668441772, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -0.3718518614768982, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.4620064496994019, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.9157564640045166, "metrics/particle/momentum/class_4/event/px_bias": 0.1680225133895874, "metrics/particle/momentum/class_4/event/py_bias": 0.028860682621598244, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.9609191417694092, "metrics/particle/momentum/class_4/event/pz_bias": 0.09551085531711578, "metrics/particle/momentum/class_4/event/pz_residual_rms": 4.02515983581543, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00399613194167614, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.013931241817772388, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01317055243998766, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006211600266396999, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.033064860850572586, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.11000760644674301, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0307103730738163, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.3927091360092163, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.08274874091148376, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0334489531815052, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030890051275491714, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.0005039134994149208, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12413144111633301, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.535963773727417, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.21475441753864288, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.3941473960876465, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.2889646291732788, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7506117820739746, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.48526105284690857, "metrics/particle/momentum/class_5/event/px_bias": 0.03928323835134506, "metrics/particle/momentum/class_5/event/py_bias": 0.011596232652664185, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4890132546424866, "metrics/particle/momentum/class_5/event/pz_bias": -0.18415111303329468, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2689878940582275, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.763561487197876, "step_valid_time": 17351.36056613922, "step_total_time": 17353.124128103256, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990226043783729, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.054177895381179675, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2907, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9641791044776119, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9616275223288124, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9629016230539914, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03837247767118756, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9625207296849088, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9599735362222958, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9612454455117588, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9641791044776119, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009201739658804, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06273645592738919, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529516200621394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9567736185383244, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9548587947520569, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04322638146167558, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9507323568575233, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9545454545454546, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95263509006004, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529516200621394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002858965305625, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06356605554212902, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1201, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9346303501945525, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9346303501945525, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9346303501945525, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06536964980544747, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9346303501945525} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_41000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_41000.json new file mode 100644 index 0000000000000000000000000000000000000000..20a582c3ced465d02dd426ac0b6c5c8d1d4a2d6f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_41000.json @@ -0,0 +1 @@ +{"epoch": 1.9423915103278377, "train": {"Classification_binary": 2.408968448638916, "Classification": 0.018429841846227646, "Regression_pt": 0.127537801861763, "Regression_eta": 0.00914840679615736, "Regression_sin_phi": 0.007892219349741936, "Regression_cos_phi": 0.007596391718834639, "Regression_energy": 0.1262587308883667, "Total": 8.252604484558105, "Total_unweighted": 2.7058318411000073}, "train_group": {}, "valid": {"Classification_binary": 2.3446594874064126, "Classification": 0.018456523617108662, "Regression_pt": 0.11845966180165608, "Regression_eta": 0.00982248658935229, "Regression_sin_phi": 0.007827013110121092, "Regression_cos_phi": 0.007808549329638481, "Regression_energy": 0.12172803282737732, "Total": 8.03374163309733, "Total_unweighted": 2.6287617546816664, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01883256807923317, "diagnostic/loss/clic_pf/Classification_binary": 2.3520607948303223, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.0074178786017000675, "diagnostic/loss/clic_pf/Regression_energy": 0.12463248521089554, "diagnostic/loss/clic_pf/Regression_eta": 0.009187577292323112, "diagnostic/loss/clic_pf/Regression_pt": 0.12138879299163818, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007358197588473558, "diagnostic/regression/clic_pf/energy_pred_mean": 0.031915053725242615, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18364812433719635, "diagnostic/regression/clic_pf/energy_residual_mean": 0.047468673437833786, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41491204500198364, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0012331336038187146, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17633233964443207, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06761140376329422, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4163253605365753, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9731051325798035, "metrics/particle/class_1/pid_accuracy": 0.9884381294250488, "metrics/particle/class_2/efficiency": 0.45816418528556824, "metrics/particle/class_2/pid_accuracy": 0.7814422845840454, "metrics/particle/class_3/efficiency": 0.7569134831428528, "metrics/particle/class_3/pid_accuracy": 0.9881253838539124, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5692307949066162, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7510204315185547, "metrics/particle/count/bias_mean": -0.7816993594169617, "metrics/particle/count/mae": 3.0627450942993164, "metrics/particle/count/prediction_mean": 69.21764373779297, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.044374991208314896, "metrics/particle/event/energy_response_mean": 1.0062204599380493, "metrics/particle/event/met_abs_error": 3.914795160293579, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03385237231850624, "metrics/particle/event/scalar_pt_response_mean": 1.0074068307876587, "metrics/particle/event/vector_pt_closure": 0.0351683534681797, "metrics/particle/matched/delta_eta_abs_mean": 0.0077553014270961285, "metrics/particle/matched/delta_phi_abs_mean": 0.00849053356796503, "metrics/particle/matched/delta_r_mean": 0.012976028025150299, "metrics/particle/matched/energy_relative_abs_mean": 0.08252599090337753, "metrics/particle/matched/pid_accuracy": 0.97306889295578, "metrics/particle/matched/pt_relative_abs_mean": 0.07226242870092392, "metrics/particle/matching/duplicate_fraction": 0.024881258606910706, "metrics/particle/matching/efficiency": 0.8365997672080994, "metrics/particle/matching/f1": 0.8412972688674927, "metrics/particle/matching/purity": 0.8460478186607361, "metrics/particle/matching/target_coverage_dr0p05": 0.8206706047058105, "metrics/particle/matching/target_coverage_dr0p10": 0.868047297000885, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7472058534622192, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8707924485206604, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014798608608543873, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07837004959583282, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07153236120939255, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014819271862506866, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1265517920255661, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0030804334674030542, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.29872629046440125, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0018045102478936315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.149695634841919, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8765.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2629793882369995, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.848517119884491, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7996744513511658, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6932761669158936, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11251556128263474, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.08209228515625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01254428829997778, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6735766530036926, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.062200430780649185, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.861097574234009, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10443.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8154246807098389, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.29102423787117004, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.017838954925537, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.1969079971313477, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.16544091701507568, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.10608203709125519, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.347322940826416, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.01553361862897873, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.579981803894043, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022752545773983, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08831122517585754, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08734802156686783, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.030286166816949844, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14368656277656555, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0192507803440094, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8789916038513184, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.13752874732017517, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.900219678878784, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1922.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6853448152542114, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.199468970298767, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.6321287155151367, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.489006757736206, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22998595237731934, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 33.18544006347656, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06179775297641754, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.337109088897705, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.050204288214445114, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.727994918823242, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2848.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6256510615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.942043662071228, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.103736877441406, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9771316647529602, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.12246586382389069, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1738954484462738, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.964208126068115, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.5475319027900696, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 14.993885040283203, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011349702253937721, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0840684175491333, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07151196151971817, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014090736396610737, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1277615875005722, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005196965066716075, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.09939312189817429, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007635199464857578, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5468629002571106, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19242.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.13189148902893066, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5606683492660522, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4148557186126709, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3213396668434143, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08419635146856308, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.096165657043457, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028793015517294407, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2264564037322998, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0030570123344659805, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3833895921707153, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21533.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8764689564704895, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.17641493678092957, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.266819953918457, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.323927402496338, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2738969027996063, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.2574443817138672, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.566863536834717, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.0768866240978241, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.0941081047058105, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013092023320496082, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08267206698656082, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07253491133451462, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.015344145707786083, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1284932941198349, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024725182447582483, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20783455669879913, "metrics/particle/momentum/all/accepted/pz_bias": 0.004451545421034098, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0580805540084839, "metrics/particle/momentum/all/accepted/samples": 29929.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21646495163440704, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6992310881614685, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7115893959999084, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6101534366607666, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10461176186800003, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.188176155090332, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010596140287816525, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5331533551216125, "metrics/particle/momentum/all/assigned/pz_bias": 0.020868200808763504, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.709261655807495, "metrics/particle/momentum/all/assigned/samples": 34824.0, "metrics/particle/momentum/all/efficiency": 0.8365898132324219, "metrics/particle/momentum/all/event/energy_bias": -0.47460442781448364, "metrics/particle/momentum/all/event/p3_error_mean": 11.617769241333008, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.472355842590332, "metrics/particle/momentum/all/event/px_bias": -0.23092184960842133, "metrics/particle/momentum/all/event/py_bias": 0.5374219417572021, "metrics/particle/momentum/all/event/pz_abs_mean": 8.960938453674316, "metrics/particle/momentum/all/event/pz_bias": 0.6399522423744202, "metrics/particle/momentum/all/event/pz_residual_rms": 16.266645431518555, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0062171961180865765, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.037327587604522705, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.018805595114827156, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003030000487342477, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05562084913253784, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012806439772248268, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.09833168238401413, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0078491335734725, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7860162854194641, "metrics/particle/momentum/class_1/accepted/samples": 16398.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03540797159075737, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.11391694098711014, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.09114070236682892, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06543149054050446, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.017306780442595482, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.0546703338623047, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0034969178959727287, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.18975187838077545, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.014728622511029243, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.6200711727142334, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9718484878540039, "metrics/particle/momentum/class_1/event/energy_bias": 2.4736807346343994, "metrics/particle/momentum/class_1/event/p3_error_mean": 8.074936866760254, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.153079032897949, "metrics/particle/momentum/class_1/event/px_bias": -0.6975193023681641, "metrics/particle/momentum/class_1/event/py_bias": -0.3134927749633789, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.138767242431641, "metrics/particle/momentum/class_1/event/pz_bias": 0.5105533003807068, "metrics/particle/momentum/class_1/event/pz_residual_rms": 12.70143985748291, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03557148203253746, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19234079122543335, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19388817250728607, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.02492791786789894, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23413291573524475, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03129657357931137, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5927090644836426, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.07825004309415817, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5960569381713867, "metrics/particle/momentum/class_2/accepted/samples": 1342.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8336725831031799, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6232782006263733, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7004559636116028, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.23655667901039124, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34716981649398804, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9480884075164795, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0728301852941513, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8982927799224854, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.07130337506532669, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.901438236236572, "metrics/particle/momentum/class_2/assigned/samples": 2650.0, "metrics/particle/momentum/class_2/efficiency": 0.4555329382419586, "metrics/particle/momentum/class_2/event/energy_bias": -7.297642230987549, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.974114418029785, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.367230415344238, "metrics/particle/momentum/class_2/event/px_bias": 0.3288900852203369, "metrics/particle/momentum/class_2/event/py_bias": 0.3993012309074402, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.708697319030762, "metrics/particle/momentum/class_2/event/pz_bias": -0.03396603465080261, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.351679801940918, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020376106724143028, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13572506606578827, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13556523621082306, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.037756871432065964, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17716023325920105, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009426685865037143, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20365159213542938, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.007656350266188383, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7033993601799011, "metrics/particle/momentum/class_3/accepted/samples": 11669.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3185756802558899, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3979578018188477, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4401055574417114, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3149327039718628, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.163572296500206, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 16.99167823791504, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007792383898049593, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5084916949272156, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.02236940898001194, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.3486850261688232, "metrics/particle/momentum/class_3/assigned/samples": 14758.0, "metrics/particle/momentum/class_3/efficiency": 0.7571372985839844, "metrics/particle/momentum/class_3/event/energy_bias": 5.370839595794678, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.226552963256836, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.369964599609375, "metrics/particle/momentum/class_3/event/px_bias": -0.07414859533309937, "metrics/particle/momentum/class_3/event/py_bias": 0.4039294719696045, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.042575836181641, "metrics/particle/momentum/class_3/event/pz_bias": 0.2125093936920166, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.955921173095703, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010707639157772064, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05151345580816269, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.051949966698884964, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.004187741782516241, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09825033694505692, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.20879992842674255, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.011881989426910877, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7559056282043457, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.05457243323326111, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.3682936728000641, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.34034737944602966, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.2917708456516266, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03867403417825699, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.293485164642334, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.21649597585201263, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.007131215650588274, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.7467795014381409, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.6263570189476013, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2111423015594482, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.805152416229248, "metrics/particle/momentum/class_4/event/px_bias": 0.15416060388088226, "metrics/particle/momentum/class_4/event/py_bias": 0.03096192143857479, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7388189435005188, "metrics/particle/momentum/class_4/event/pz_bias": 0.13147090375423431, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.90557861328125, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.003972610924392939, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.0148488599807024, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014664492569863796, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005541641730815172, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04089563712477684, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12651483714580536, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03996540233492851, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5409404635429382, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05628879740834236, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06563446670770645, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.055886998772621155, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03157820552587509, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.4124857187271118, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5597333908081055, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.21036967635154724, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.384034633636475, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.39512547850608826, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7670828104019165, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.49549123644828796, "metrics/particle/momentum/class_5/event/px_bias": 0.05769531428813934, "metrics/particle/momentum/class_5/event/py_bias": 0.01672203838825226, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4974271357059479, "metrics/particle/momentum/class_5/event/pz_bias": -0.1806153655052185, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2815330028533936, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.767354965209961, "step_valid_time": 17787.02538561821, "step_total_time": 17788.79274082184, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003250224609816, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05626162861710948, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2899, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9638474295190713, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9603436880370125, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9620923688131104, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03965631196298744, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9615257048092869, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.958030403172505, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9597748717099818, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9638474295190713, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013974004230435, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06540629757862226, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2140, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9571428571428572, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9543734698419765, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04285714285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9498446515756769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553571428571429, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9525929223236145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516200621393698, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020059267178012, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06288808403685853, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9369649805447471, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9355089355089355, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9362363919129082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0644910644910645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9316239316239316, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.932348367029549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9369649805447471} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_42000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_42000.json new file mode 100644 index 0000000000000000000000000000000000000000..12a6516a4af745afc78e15961d0e861621c786ff --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_42000.json @@ -0,0 +1 @@ +{"epoch": 1.9897669130187607, "train": {"Classification_binary": 2.4017722606658936, "Classification": 0.018298698589205742, "Regression_pt": 0.1354491263628006, "Regression_eta": 0.006953280884772539, "Regression_sin_phi": 0.007192665245383978, "Regression_cos_phi": 0.006486326456069946, "Regression_energy": 0.13647392392158508, "Total": 8.278342247009277, "Total_unweighted": 2.7126262821257114}, "train_group": {}, "valid": {"Classification_binary": 2.3344736099243164, "Classification": 0.01839925100406011, "Regression_pt": 0.11824764808019002, "Regression_eta": 0.009879237661759058, "Regression_sin_phi": 0.007800657923022906, "Regression_cos_phi": 0.007789730404814084, "Regression_energy": 0.12149200836817424, "Total": 8.014731725056967, "Total_unweighted": 2.618082143366337, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018882326781749725, "diagnostic/loss/clic_pf/Classification_binary": 2.338555335998535, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007391481194645166, "diagnostic/loss/clic_pf/Regression_energy": 0.12405240535736084, "diagnostic/loss/clic_pf/Regression_eta": 0.009057569317519665, "diagnostic/loss/clic_pf/Regression_pt": 0.12104029953479767, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007360814139246941, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03190136328339577, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18212054669857025, "diagnostic/regression/clic_pf/energy_residual_mean": 0.047454990446567535, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4120251536369324, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0012430890928953886, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17479541897773743, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06760144978761673, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4141713082790375, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9748544096946716, "metrics/particle/class_1/pid_accuracy": 0.9880306720733643, "metrics/particle/class_2/efficiency": 0.46493008732795715, "metrics/particle/class_2/pid_accuracy": 0.7870482802391052, "metrics/particle/class_3/efficiency": 0.761741578578949, "metrics/particle/class_3/pid_accuracy": 0.987323522567749, "metrics/particle/class_4/efficiency": 0.9418604373931885, "metrics/particle/class_4/pid_accuracy": 0.5390946269035339, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7571428418159485, "metrics/particle/count/bias_mean": -0.4032679796218872, "metrics/particle/count/mae": 2.945751667022705, "metrics/particle/count/prediction_mean": 69.59607696533203, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.044189997017383575, "metrics/particle/event/energy_response_mean": 1.0118108987808228, "metrics/particle/event/met_abs_error": 3.9545483589172363, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03377053886651993, "metrics/particle/event/scalar_pt_response_mean": 1.0099589824676514, "metrics/particle/event/vector_pt_closure": 0.0356580913066864, "metrics/particle/matched/delta_eta_abs_mean": 0.007134161423891783, "metrics/particle/matched/delta_phi_abs_mean": 0.008576051332056522, "metrics/particle/matched/delta_r_mean": 0.012555534020066261, "metrics/particle/matched/energy_relative_abs_mean": 0.08313678205013275, "metrics/particle/matched/pid_accuracy": 0.9724346399307251, "metrics/particle/matched/pt_relative_abs_mean": 0.07242576032876968, "metrics/particle/matching/duplicate_fraction": 0.026088915765285492, "metrics/particle/matching/efficiency": 0.8400452136993408, "metrics/particle/matching/f1": 0.8424719572067261, "metrics/particle/matching/purity": 0.8449127674102783, "metrics/particle/matching/target_coverage_dr0p05": 0.8241440057754517, "metrics/particle/matching/target_coverage_dr0p10": 0.8710258603096008, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7508847117424011, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8752369284629822, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014940119348466396, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07916800677776337, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07173077017068863, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.015489003621041775, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12675583362579346, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031843511387705803, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31317195296287537, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008483952842652798, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2419264316558838, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8793.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26137420535087585, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9586464166641235, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8621184825897217, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7589300870895386, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1136016771197319, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.948381423950195, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.012304464355111122, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6679993271827698, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.038982316851615906, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.531497001647949, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10484.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8180295825004578, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.5778517127037048, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.2308220863342285, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2718303203582764, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.2496454268693924, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.08563345670700073, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.512132167816162, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.5069393515586853, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.778302192687988, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.02317359857261181, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08707646280527115, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08344938606023788, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01974552311003208, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13811711966991425, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021886399015784264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9246684312820435, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.052980463951826096, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0768773555755615, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1919.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6917332410812378, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.274612545967102, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.878999948501587, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.7378122806549072, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2283712774515152, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.879024505615234, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06549912691116333, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4552788734436035, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.18892347812652588, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.827569007873535, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2855.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6246744990348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.012445002794265747, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.125640392303467, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0166926383972168, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.09784753620624542, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08704017102718353, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.96723747253418, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.21087884902954102, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.62270736694336, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010570406913757324, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08481703698635101, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07196695357561111, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016270063817501068, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12853650748729706, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005686223739758134, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1022881418466568, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004163135774433613, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6016039848327637, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19345.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1321379691362381, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7245957255363464, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.509220540523529, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.41531607508659363, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08109109103679657, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.249439239501953, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0028713007923215628, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23072506487369537, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.002873091259971261, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5594602823257446, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21593.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8811606168746948, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.36545228958129883, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.968279838562012, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.151625156402588, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.22803618013858795, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.11510981619358063, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3365793228149414, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.05721212923526764, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.326364994049072, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012653395533561707, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08330870419740677, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07263096421957016, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016263462603092194, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1286536157131195, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0026948796585202217, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2164861410856247, "metrics/particle/momentum/all/accepted/pz_bias": -0.0017788116820156574, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1351608037948608, "metrics/particle/momentum/all/accepted/samples": 30057.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21666096150875092, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8397934436798096, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.8088170886039734, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.7082616686820984, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10288560390472412, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 14.57599925994873, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.0108210239559412, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.543775737285614, "metrics/particle/momentum/all/assigned/pz_bias": 0.025364378467202187, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9041874408721924, "metrics/particle/momentum/all/assigned/samples": 34932.0, "metrics/particle/momentum/all/efficiency": 0.8401677012443542, "metrics/particle/momentum/all/event/energy_bias": 0.9308590292930603, "metrics/particle/momentum/all/event/p3_error_mean": 11.655024528503418, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.448880672454834, "metrics/particle/momentum/all/event/px_bias": -0.07623828947544098, "metrics/particle/momentum/all/event/py_bias": 0.057563815265893936, "metrics/particle/momentum/all/event/pz_abs_mean": 8.967445373535156, "metrics/particle/momentum/all/event/pz_bias": 0.6606060266494751, "metrics/particle/momentum/all/event/pz_residual_rms": 17.141189575195312, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006453561596572399, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.038541849702596664, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01947369985282421, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0027148211374878883, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05769789591431618, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0016435354482382536, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10746245831251144, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0032779944594949484, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9020723104476929, "metrics/particle/momentum/class_1/accepted/samples": 16428.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03431679680943489, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09041208028793335, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.0794970840215683, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.05378085374832153, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016300160437822342, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.7642879486083984, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003319305367767811, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.18591289222240448, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.010912713594734669, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9926382303237915, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9736264944076538, "metrics/particle/momentum/class_1/event/energy_bias": 1.3479584455490112, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.989485740661621, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.099701404571533, "metrics/particle/momentum/class_1/event/px_bias": -0.45813703536987305, "metrics/particle/momentum/class_1/event/py_bias": -0.2541659474372864, "metrics/particle/momentum/class_1/event/pz_abs_mean": 6.03373908996582, "metrics/particle/momentum/class_1/event/pz_bias": 0.8155322074890137, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.70730972290039, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03508610650897026, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19205988943576813, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19306223094463348, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.029474256560206413, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23253333568572998, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02900652587413788, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6168138980865479, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.03926076367497444, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6441850662231445, "metrics/particle/momentum/class_2/accepted/samples": 1379.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8609112501144409, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6533251404762268, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7311761975288391, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.2729266881942749, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33345726132392883, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.7167859077453613, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07100372016429901, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8981094360351562, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.15403245389461517, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.034147262573242, "metrics/particle/momentum/class_2/assigned/samples": 2690.0, "metrics/particle/momentum/class_2/efficiency": 0.46809232234954834, "metrics/particle/momentum/class_2/event/energy_bias": -5.989552974700928, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.271234512329102, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.400027275085449, "metrics/particle/momentum/class_2/event/px_bias": 0.36303627490997314, "metrics/particle/momentum/class_2/event/py_bias": -0.2848208546638489, "metrics/particle/momentum/class_2/event/pz_abs_mean": 9.055195808410645, "metrics/particle/momentum/class_2/event/pz_bias": -0.18798303604125977, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.93510627746582, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018871689215302467, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13519783318042755, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13441331684589386, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.04114934056997299, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17638812959194183, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011930123437196016, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2055094838142395, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.0037545147351920605, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7597841620445251, "metrics/particle/momentum/class_3/accepted/samples": 11735.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3130688965320587, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.7390499114990234, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.670741081237793, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.547595739364624, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16193431615829468, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 22.226118087768555, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008700343780219555, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5307244658470154, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.0256255604326725, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.477670907974243, "metrics/particle/momentum/class_3/assigned/samples": 14827.0, "metrics/particle/momentum/class_3/efficiency": 0.7614196538925171, "metrics/particle/momentum/class_3/event/energy_bias": 6.312108039855957, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.623366355895996, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.473477363586426, "metrics/particle/momentum/class_3/event/px_bias": -0.21861141920089722, "metrics/particle/momentum/class_3/event/py_bias": 0.49475574493408203, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.377340793609619, "metrics/particle/momentum/class_3/event/pz_bias": 0.2158319503068924, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.353690147399902, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010985122062265873, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05020533874630928, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0508260652422905, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0038541534449905157, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09247028082609177, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22411802411079407, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.001556798000819981, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7946471571922302, "metrics/particle/momentum/class_4/accepted/samples": 338.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06252793967723846, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.7249521613121033, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.46390673518180847, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.4183228313922882, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 4.5287089347839355, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.27840542793273926, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.043311476707458496, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8960316181182861, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9337016344070435, "metrics/particle/momentum/class_4/event/energy_bias": -0.4168594479560852, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.3493627309799194, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8901658058166504, "metrics/particle/momentum/class_4/event/px_bias": 0.1633770763874054, "metrics/particle/momentum/class_4/event/py_bias": 0.0808527022600174, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.8381403088569641, "metrics/particle/momentum/class_4/event/pz_bias": 0.01707359403371811, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.1684865951538086, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004226086661219597, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.013996347784996033, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.013572490774095058, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0038788930978626013, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.03704725205898285, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1386362761259079, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.048534538596868515, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5926165580749512, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.049908898770809174, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.030500903725624084, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030346542596817017, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.010183349251747131, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12145071476697922, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5103719234466553, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.42142143845558167, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.211730480194092, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.3227948546409607, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7602574825286865, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.4985697269439697, "metrics/particle/momentum/class_5/event/px_bias": 0.07409681379795074, "metrics/particle/momentum/class_5/event/py_bias": 0.020942185074090958, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.490239679813385, "metrics/particle/momentum/class_5/event/pz_bias": -0.19984863698482513, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2487659454345703, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7047619819641113, "step_valid_time": 18223.93243598938, "step_total_time": 18225.63719844818, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001424974025035, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.053547272571192295, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9618573797678275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9596293845135672, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607420904422727, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04037061548643282, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9583057577763071, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9594169289382144, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9618573797678275, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040462637486174, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06652520295939102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2152, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9551708832667555, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9585746102449889, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568697198755002, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04142538975501114, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533954727030626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956792873051225, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9550911516229436, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9551708832667555, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004694267735173, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06411941711684499, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385214007782101, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392523364485982, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388867263526665, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06074766355140187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9361867704280156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9369158878504673, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9365511872323862, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385214007782101} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_43000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_43000.json new file mode 100644 index 0000000000000000000000000000000000000000..6956683e568e6612d42781d2959ad5a838ff6e54 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_43000.json @@ -0,0 +1 @@ +{"epoch": 2.0371423157096835, "train": {"Classification_binary": 2.2820258140563965, "Classification": 0.019248371943831444, "Regression_pt": 0.12880529463291168, "Regression_eta": 0.010192899964749813, "Regression_sin_phi": 0.00778765557333827, "Regression_cos_phi": 0.008123988285660744, "Regression_energy": 0.13187669217586517, "Total": 8.337272644042969, "Total_unweighted": 2.5880607166327536}, "train_group": {}, "valid": {"Classification_binary": 2.333029588063558, "Classification": 0.018389215071996052, "Regression_pt": 0.11775258183479309, "Regression_eta": 0.009834999839464823, "Regression_sin_phi": 0.007794932772715886, "Regression_cos_phi": 0.007749072586496671, "Regression_energy": 0.12093212207158406, "Total": 7.992462793986003, "Total_unweighted": 2.6154825122406087, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01892186515033245, "diagnostic/loss/clic_pf/Classification_binary": 2.3308916091918945, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007337033282965422, "diagnostic/loss/clic_pf/Regression_energy": 0.1248021274805069, "diagnostic/loss/clic_pf/Regression_eta": 0.009150685742497444, "diagnostic/loss/clic_pf/Regression_pt": 0.12172354757785797, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007360484451055527, "diagnostic/regression/clic_pf/energy_pred_mean": 0.026407385244965553, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18273061513900757, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04196101054549217, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4153771996498108, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0048754857853055, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17585347592830658, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06396904587745667, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4172355830669403, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9736418128013611, "metrics/particle/class_1/pid_accuracy": 0.9881995320320129, "metrics/particle/class_2/efficiency": 0.46087053418159485, "metrics/particle/class_2/pid_accuracy": 0.7917788028717041, "metrics/particle/class_3/efficiency": 0.7627330422401428, "metrics/particle/class_3/pid_accuracy": 0.9866052269935608, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5446153879165649, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7540650367736816, "metrics/particle/count/bias_mean": -0.28431373834609985, "metrics/particle/count/mae": 2.9156863689422607, "metrics/particle/count/prediction_mean": 69.71503448486328, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04306089133024216, "metrics/particle/event/energy_response_mean": 1.0085670948028564, "metrics/particle/event/met_abs_error": 4.078310489654541, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03354630246758461, "metrics/particle/event/scalar_pt_response_mean": 1.0115560293197632, "metrics/particle/event/vector_pt_closure": 0.03573434427380562, "metrics/particle/matched/delta_eta_abs_mean": 0.008008650504052639, "metrics/particle/matched/delta_phi_abs_mean": 0.00882317591458559, "metrics/particle/matched/delta_r_mean": 0.013397016562521458, "metrics/particle/matched/energy_relative_abs_mean": 0.08212485164403915, "metrics/particle/matched/pid_accuracy": 0.9725539684295654, "metrics/particle/matched/pt_relative_abs_mean": 0.07217809557914734, "metrics/particle/matching/duplicate_fraction": 0.025800645351409912, "metrics/particle/matching/efficiency": 0.8396157026290894, "metrics/particle/matching/f1": 0.8413242697715759, "metrics/particle/matching/purity": 0.8430398106575012, "metrics/particle/matching/target_coverage_dr0p05": 0.8194661140441895, "metrics/particle/matching/target_coverage_dr0p10": 0.8695412874221802, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7516409754753113, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8762266635894775, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015963314101099968, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07769883424043655, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07130375504493713, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.012642579153180122, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12635734677314758, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.00296363839879632, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.30500590801239014, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0033132433891296387, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1687579154968262, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8773.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25916358828544617, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8725200295448303, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8874664306640625, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.7802584767341614, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11381183564662933, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.989447593688965, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011724335141479969, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6503039002418518, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.017641764134168625, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.3866195678710938, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10491.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8161689639091492, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.007155277766287327, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.065280437469482, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2436089515686035, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.30504554510116577, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.02502291090786457, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.358236312866211, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.17777329683303833, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.345863342285156, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023848213255405426, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08368301391601562, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0825115293264389, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01947479508817196, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13637115061283112, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02106371708214283, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8598330020904541, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05945910885930061, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.85223650932312, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1899.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6744309067726135, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.315467357635498, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 31286.375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 31286.224609375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23486429452896118, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1677084.875, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06680584698915482, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4529953002929688, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1803600788116455, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.983618259429932, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2874.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6181640625, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7818291783332825, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.231950759887695, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.025180697441101, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.003497377038002014, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12095767259597778, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.074873924255371, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4853995442390442, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.64398956298828, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.011278962716460228, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0843300148844719, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07213139533996582, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.01112153846770525, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12877899408340454, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007237386307679117, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10364404320716858, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.008533776737749577, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6012113690376282, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19344.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1356925219297409, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6975181102752686, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4873705804347992, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3891957700252533, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08001111447811127, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.422330379486084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002778163645416498, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.23198285698890686, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.007155818864703178, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.7729154825210571, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21597.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8811150789260864, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.0253667831420898, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.290204048156738, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.410639762878418, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.002207621932029724, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.048899903893470764, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.540590286254883, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.09421192109584808, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 6.994745254516602, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.013443302363157272, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08235093951225281, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07254620641469955, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.012094583362340927, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12857218086719513, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002665245207026601, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21033881604671478, "metrics/particle/momentum/all/accepted/pz_bias": -0.002706277184188366, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.070924162864685, "metrics/particle/momentum/all/accepted/samples": 30016.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21702851355075836, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8008283972740173, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 2572.41796875, "metrics/particle/momentum/all/assigned/pt_relative_bias": 2572.3125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10288313031196594, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 480839.65625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010725931264460087, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5400829911231995, "metrics/particle/momentum/all/assigned/pz_bias": 0.02454032190144062, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9816672801971436, "metrics/particle/momentum/all/assigned/samples": 34962.0, "metrics/particle/momentum/all/efficiency": 0.8390216827392578, "metrics/particle/momentum/all/event/energy_bias": 0.23638233542442322, "metrics/particle/momentum/all/event/p3_error_mean": 11.571533203125, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.637697219848633, "metrics/particle/momentum/all/event/px_bias": 0.30633530020713806, "metrics/particle/momentum/all/event/py_bias": 0.14483466744422913, "metrics/particle/momentum/all/event/pz_abs_mean": 8.780303955078125, "metrics/particle/momentum/all/event/pz_bias": 0.2134142965078354, "metrics/particle/momentum/all/event/pz_residual_rms": 18.03362464904785, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006776501890271902, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03829869255423546, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01992477849125862, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003869391046464443, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05945168808102608, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001705964794382453, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10673213005065918, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0038175275549292564, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8360951542854309, "metrics/particle/momentum/class_1/accepted/samples": 16413.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03412080556154251, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10611465573310852, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.11139775067567825, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08596397191286087, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016001896932721138, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.475803375244141, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003674509469419718, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.19486187398433685, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005397344473749399, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.039855480194092, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9727374911308289, "metrics/particle/momentum/class_1/event/energy_bias": 1.9816433191299438, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.94051456451416, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 4.093820571899414, "metrics/particle/momentum/class_1/event/px_bias": -0.4043322503566742, "metrics/particle/momentum/class_1/event/py_bias": -0.522113561630249, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.989500522613525, "metrics/particle/momentum/class_1/event/pz_bias": 0.33331337571144104, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.166393280029297, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03624724969267845, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18988485634326935, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1914302110671997, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.017644992098212242, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23163361847400665, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.02861335314810276, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5677238702774048, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.058702096343040466, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.519012928009033, "metrics/particle/momentum/class_2/accepted/samples": 1363.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.836542546749115, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.7661141157150269, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 33436.3046875, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 33435.83984375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34139084815979004, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1733816.125, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06805504113435745, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9174275398254395, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10299436002969742, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.705960750579834, "metrics/particle/momentum/class_2/assigned/samples": 2689.0, "metrics/particle/momentum/class_2/efficiency": 0.4626612365245819, "metrics/particle/momentum/class_2/event/energy_bias": -5.8548126220703125, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.963516235351562, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.2322492599487305, "metrics/particle/momentum/class_2/event/px_bias": 0.5850021243095398, "metrics/particle/momentum/class_2/event/py_bias": -0.03200159966945648, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.763102531433105, "metrics/particle/momentum/class_2/event/pz_bias": -0.3254123330116272, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.602805137634277, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.020347697660326958, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13342270255088806, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13385893404483795, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.027290532365441322, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1755724996328354, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011092149652540684, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19818715751171112, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.006244800519198179, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7034103870391846, "metrics/particle/momentum/class_3/accepted/samples": 11720.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3184868097305298, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5889097452163696, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.631309151649475, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4977941513061523, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.16107970476150513, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 25.117063522338867, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008616046980023384, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.5096116065979004, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03891656547784805, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.2842812538146973, "metrics/particle/momentum/class_3/assigned/samples": 14856.0, "metrics/particle/momentum/class_3/efficiency": 0.7604464292526245, "metrics/particle/momentum/class_3/event/energy_bias": 5.049480438232422, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.652223587036133, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.574859142303467, "metrics/particle/momentum/class_3/event/px_bias": -0.06587405502796173, "metrics/particle/momentum/class_3/event/py_bias": 0.618583083152771, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.355807304382324, "metrics/particle/momentum/class_3/event/pz_bias": 0.301566481590271, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.436700820922852, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010488199070096016, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05297454446554184, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05324176698923111, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007366814184933901, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10132666677236557, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22613447904586792, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.031204013153910637, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7805327773094177, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06075305864214897, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.48469877243042, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.4808210134506226, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.4353011846542358, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03591160103678703, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 23.89559555053711, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.24160116910934448, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.019528627395629883, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.7950714230537415, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.6525753736495972, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2323777675628662, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8229807615280151, "metrics/particle/momentum/class_4/event/px_bias": 0.11427463591098785, "metrics/particle/momentum/class_4/event/py_bias": 0.06616935133934021, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7355278134346008, "metrics/particle/momentum/class_4/event/pz_bias": 0.15319885313510895, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.8963780403137207, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004598705098032951, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.014417535625398159, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014201469719409943, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006785119418054819, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.037386659532785416, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13904304802417755, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0830138549208641, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6041698455810547, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05019490793347359, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03132690116763115, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03097611479461193, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013037947937846184, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11731210350990295, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.501452624797821, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.445701003074646, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.187251567840576, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.2873530387878418, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7809392809867859, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5177675485610962, "metrics/particle/momentum/class_5/event/px_bias": 0.07726487517356873, "metrics/particle/momentum/class_5/event/py_bias": 0.014197465032339096, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5015902519226074, "metrics/particle/momentum/class_5/event/pz_bias": -0.249252051115036, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.263247013092041, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.8522515296936035, "step_valid_time": 18660.791980981827, "step_total_time": 18662.64423274994, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0012296920152193, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05387638026015451, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3026, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625207296849088, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9590218109715797, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9607680847541797, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.040978189028420355, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9570389953734303, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9587816586657838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625207296849088, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025225641585862, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06445761725215382, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2139, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9494007989347537, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9527839643652561, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.951089373054691, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04721603563474387, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9510022271714922, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9493108048021343, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9494007989347537, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005317345722586, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06091128208497332, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1193, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1189, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9284046692607004, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9276827371695179, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9280435628160248, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07231726283048212, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9252918287937744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9245723172628305, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9249319330999611, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9284046692607004} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_44000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_44000.json new file mode 100644 index 0000000000000000000000000000000000000000..1672a7d8df34670c55a22682aaa27497a2032adb --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_44000.json @@ -0,0 +1 @@ +{"epoch": 2.0845177184006065, "train": {"Classification_binary": 2.3340532779693604, "Classification": 0.018032638356089592, "Regression_pt": 0.11651170998811722, "Regression_eta": 0.00958644412457943, "Regression_sin_phi": 0.007356188725680113, "Regression_cos_phi": 0.00770972203463316, "Regression_energy": 0.11737386137247086, "Total": 7.870797634124756, "Total_unweighted": 2.6106238425709307}, "train_group": {}, "valid": {"Classification_binary": 2.320399443308512, "Classification": 0.01828324794769287, "Regression_pt": 0.11768651008605957, "Regression_eta": 0.009669229388237, "Regression_sin_phi": 0.0077241479108730955, "Regression_cos_phi": 0.007688986758391063, "Regression_energy": 0.12077446778615315, "Total": 7.956026077270508, "Total_unweighted": 2.602226033185919, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018735390156507492, "diagnostic/loss/clic_pf/Classification_binary": 2.3215532302856445, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007282518781721592, "diagnostic/loss/clic_pf/Regression_energy": 0.12350775301456451, "diagnostic/loss/clic_pf/Regression_eta": 0.009145714342594147, "diagnostic/loss/clic_pf/Regression_pt": 0.12018047273159027, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007309705018997192, "diagnostic/regression/clic_pf/energy_pred_mean": 0.025243880227208138, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.17956838011741638, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04079750180244446, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4069361686706543, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.009579446166753769, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17187857627868652, "diagnostic/regression/clic_pf/pt_residual_mean": 0.05926508828997612, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4071177840232849, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9741387963294983, "metrics/particle/class_1/pid_accuracy": 0.988307535648346, "metrics/particle/class_2/efficiency": 0.46335136890411377, "metrics/particle/class_2/pid_accuracy": 0.7980043888092041, "metrics/particle/class_3/efficiency": 0.7659661769866943, "metrics/particle/class_3/pid_accuracy": 0.9872527122497559, "metrics/particle/class_4/efficiency": 0.9437984228134155, "metrics/particle/class_4/pid_accuracy": 0.558521568775177, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7612245082855225, "metrics/particle/count/bias_mean": -0.47647058963775635, "metrics/particle/count/mae": 3.0019607543945312, "metrics/particle/count/prediction_mean": 69.52287292480469, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04351189732551575, "metrics/particle/event/energy_response_mean": 1.0021411180496216, "metrics/particle/event/met_abs_error": 3.803166389465332, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03258730471134186, "metrics/particle/event/scalar_pt_response_mean": 1.0025662183761597, "metrics/particle/event/vector_pt_closure": 0.034601904451847076, "metrics/particle/matched/delta_eta_abs_mean": 0.006967443041503429, "metrics/particle/matched/delta_phi_abs_mean": 0.008319827727973461, "metrics/particle/matched/delta_r_mean": 0.012222924269735813, "metrics/particle/matched/energy_relative_abs_mean": 0.08233878761529922, "metrics/particle/matched/pid_accuracy": 0.9733343720436096, "metrics/particle/matched/pt_relative_abs_mean": 0.07188484817743301, "metrics/particle/matching/duplicate_fraction": 0.024913039058446884, "metrics/particle/matching/efficiency": 0.8414270877838135, "metrics/particle/matching/f1": 0.8443005681037903, "metrics/particle/matching/purity": 0.8471937775611877, "metrics/particle/matching/target_coverage_dr0p05": 0.8266463875770569, "metrics/particle/matching/target_coverage_dr0p10": 0.8719035387039185, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7534524202346802, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8763387203216553, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014508518390357494, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07809574902057648, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07073386013507843, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01149037480354309, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12504348158836365, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0027213969733566046, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.295571506023407, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0038827285170555115, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.179203987121582, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8819.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2528790533542633, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7383302450180054, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7558829188346863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6449500918388367, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11013384163379669, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.320585250854492, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010803059674799442, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6337774991989136, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.014631805010139942, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.4248406887054443, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10460.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8204483985900879, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.7067781686782837, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.346023082733154, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.4183850288391113, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3532565236091614, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.0049895793199539185, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.536151885986328, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.19317451119422913, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.102168083190918, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022802814841270447, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08501381427049637, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08218012005090714, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.013452814891934395, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.134707510471344, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.02022821642458439, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8638195395469666, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1630493700504303, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.794839382171631, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1928.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6463635563850403, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.538935899734497, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 46751.27734375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 46751.125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2256482094526291, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 2497382.25, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06517168879508972, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.489499568939209, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.2316594272851944, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.240145683288574, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2854.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6276041865348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.8309108018875122, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.412169933319092, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0367097854614258, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.02641899883747101, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.20155812799930573, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.274631500244141, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.13919997215271, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.97288703918457, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010357615537941456, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.083958201110363, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07146573066711426, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015589792281389236, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12760938704013824, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005682698683813214, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10156875848770142, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.0028246408328413963, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.578682005405426, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19357.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12793871760368347, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.614475429058075, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.40183889865875244, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3089907467365265, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08018168061971664, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.417702674865723, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0027808677405118942, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2252892553806305, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0029691916424781084, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5430169105529785, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21576.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8817071914672852, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.2508571445941925, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.889846324920654, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.082566261291504, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.05313190072774887, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.008425276726484299, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.321901798248291, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.17507080733776093, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.587427616119385, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01237067487090826, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08230839669704437, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07193753123283386, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01425199955701828, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1273321956396103, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0024581451434642076, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2072201520204544, "metrics/particle/momentum/all/accepted/pz_bias": 0.009763631969690323, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.059688925743103, "metrics/particle/momentum/all/accepted/samples": 30104.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.207802876830101, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7272278666496277, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 3824.7265625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 3824.623779296875, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10106047242879868, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 714268.5625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010289480909705162, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5329660773277283, "metrics/particle/momentum/all/assigned/pz_bias": 0.025172483175992966, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9646825790405273, "metrics/particle/momentum/all/assigned/samples": 34890.0, "metrics/particle/momentum/all/efficiency": 0.8414815068244934, "metrics/particle/momentum/all/event/energy_bias": -1.788546085357666, "metrics/particle/momentum/all/event/p3_error_mean": 11.689023971557617, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.410908222198486, "metrics/particle/momentum/all/event/px_bias": 0.2737056016921997, "metrics/particle/momentum/all/event/py_bias": 0.19812244176864624, "metrics/particle/momentum/all/event/pz_abs_mean": 9.062440872192383, "metrics/particle/momentum/all/event/pz_bias": 0.7709547281265259, "metrics/particle/momentum/all/event/pz_residual_rms": 17.555761337280273, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006275903433561325, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03758649528026581, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.019304268062114716, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004221591632813215, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05646360665559769, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013396663125604391, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10001993179321289, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0036297556944191456, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8006211519241333, "metrics/particle/momentum/class_1/accepted/samples": 16422.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.031897418200969696, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10652364045381546, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08910427987575531, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06515557318925858, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016536273062229156, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.0553572177886963, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0034376480616629124, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.18951445817947388, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.014620685949921608, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0460944175720215, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9732708930969238, "metrics/particle/momentum/class_1/event/energy_bias": 2.1810317039489746, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.883688926696777, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.965986728668213, "metrics/particle/momentum/class_1/event/px_bias": -0.2682650685310364, "metrics/particle/momentum/class_1/event/py_bias": -0.18452215194702148, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.998242378234863, "metrics/particle/momentum/class_1/event/pz_bias": 0.48272114992141724, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.95837116241455, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03529983386397362, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.18934524059295654, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1901959329843521, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03151915222406387, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.22920234501361847, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.028550513088703156, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5382274389266968, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.1469239443540573, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.431076765060425, "metrics/particle/momentum/class_2/accepted/samples": 1366.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8424066305160522, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6326987743377686, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 49579.71484375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 49579.23046875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34150874614715576, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2571906.5, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07097733020782471, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.925852060317993, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08399266004562378, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.288553714752197, "metrics/particle/momentum/class_2/assigned/samples": 2691.0, "metrics/particle/momentum/class_2/efficiency": 0.46367955207824707, "metrics/particle/momentum/class_2/event/energy_bias": -7.0142927169799805, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.97249698638916, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.297624588012695, "metrics/particle/momentum/class_2/event/px_bias": 0.602993369102478, "metrics/particle/momentum/class_2/event/py_bias": -0.08186904340982437, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.747794151306152, "metrics/particle/momentum/class_2/event/pz_bias": -0.3137555420398712, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.545073509216309, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018368620425462723, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13400070369243622, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13285744190216064, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03374733030796051, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17470349371433258, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011021619429811835, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2033725082874298, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.003573754569515586, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7671912312507629, "metrics/particle/momentum/class_3/accepted/samples": 11795.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.29843154549598694, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4656124114990234, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.627726674079895, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.5007017850875854, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15639586746692657, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 36.85356140136719, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0073056891560554504, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4968278408050537, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03276510164141655, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.4615702629089355, "metrics/particle/momentum/class_3/assigned/samples": 14783.0, "metrics/particle/momentum/class_3/efficiency": 0.7653127312660217, "metrics/particle/momentum/class_3/event/energy_bias": 3.923163414001465, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.338566780090332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.401076316833496, "metrics/particle/momentum/class_3/event/px_bias": -0.22740596532821655, "metrics/particle/momentum/class_3/event/py_bias": 0.395538866519928, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.118585586547852, "metrics/particle/momentum/class_3/event/pz_bias": 0.6784072518348694, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.241263389587402, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01090930961072445, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05427224189043045, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.055226292461156845, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.009755807928740978, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10297171026468277, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2098175585269928, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.0053240032866597176, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7430000901222229, "metrics/particle/momentum/class_4/accepted/samples": 344.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.06755253672599792, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5551925301551819, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.3847191333770752, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3401566445827484, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03591160103678703, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.24444580078125, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.23970244824886322, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.0019414662383496761, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8062714338302612, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.950276255607605, "metrics/particle/momentum/class_4/event/energy_bias": -0.6248080730438232, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.1179314851760864, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.7547364830970764, "metrics/particle/momentum/class_4/event/px_bias": 0.0865517109632492, "metrics/particle/momentum/class_4/event/py_bias": 0.05693256855010986, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.6659016013145447, "metrics/particle/momentum/class_4/event/pz_bias": 0.10497066378593445, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.4350686073303223, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004031986929476261, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015327053144574165, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015445331111550331, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007709840778261423, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.043041571974754333, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13250520825386047, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.0585620254278183, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6079333424568176, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04937109723687172, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03292258828878403, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03194792568683624, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.013703014701604843, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12505637109279633, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5101683735847473, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.43684521317481995, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.233002185821533, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.253640741109848, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7701187133789062, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5022690296173096, "metrics/particle/momentum/class_5/event/px_bias": 0.07983154058456421, "metrics/particle/momentum/class_5/event/py_bias": 0.012042216956615448, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4974132180213928, "metrics/particle/momentum/class_5/event/pz_bias": -0.18138879537582397, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2645621299743652, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.6701767444610596, "step_valid_time": 19094.54671406746, "step_total_time": 19096.21689105034, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00031690131886, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05270380487479587, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3023, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2904, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.96318407960199, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9606351306649025, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9619079165286518, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.039364869335097585, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9579887528944757, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9592580324610799, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.96318407960199, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004706009008955, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06513062236103329, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2243, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2151, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2148, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9547270306258322, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958983504235399, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9568505338078291, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04101649576460098, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9533954727030626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9576460098082925, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9555160142348754, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9547270306258322, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9991401177609249, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0613573139603556, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1282, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1204, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9369649805447471, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9391575663026521, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9380599922088041, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.060842433697347896, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9352574102964119, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.934164394234515, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9369649805447471} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_45000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_45000.json new file mode 100644 index 0000000000000000000000000000000000000000..b7cc8d0cced41e27232e36a6e53b387d284d3d94 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_45000.json @@ -0,0 +1 @@ +{"epoch": 2.131893121091529, "train": {"Classification_binary": 2.3346059322357178, "Classification": 0.01673949882388115, "Regression_pt": 0.11404215544462204, "Regression_eta": 0.007010416127741337, "Regression_sin_phi": 0.008026789873838425, "Regression_cos_phi": 0.008447648957371712, "Regression_energy": 0.11205495148897171, "Total": 7.667407989501953, "Total_unweighted": 2.600927392952144}, "train_group": {}, "valid": {"Classification_binary": 2.3222190539042153, "Classification": 0.018238031615813572, "Regression_pt": 0.11757055918375652, "Regression_eta": 0.009768102938930193, "Regression_sin_phi": 0.007708665604392688, "Regression_cos_phi": 0.007754389196634293, "Regression_energy": 0.12068116664886475, "Total": 7.961254119873047, "Total_unweighted": 2.6039399690926075, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018713701516389847, "diagnostic/loss/clic_pf/Classification_binary": 2.330080509185791, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.0073208254761993885, "diagnostic/loss/clic_pf/Regression_energy": 0.12321360409259796, "diagnostic/loss/clic_pf/Regression_eta": 0.0091124027967453, "diagnostic/loss/clic_pf/Regression_pt": 0.12018492072820663, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007319969590753317, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03217868134379387, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18149608373641968, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04773230478167534, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41144782304763794, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0013007116504013538, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17425985634326935, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06754382699728012, "diagnostic/regression/clic_pf/pt_residual_rms": 0.41263121366500854, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9747351408004761, "metrics/particle/class_1/pid_accuracy": 0.9882331490516663, "metrics/particle/class_2/efficiency": 0.4636896848678589, "metrics/particle/class_2/pid_accuracy": 0.7874513864517212, "metrics/particle/class_3/efficiency": 0.7640693783760071, "metrics/particle/class_3/pid_accuracy": 0.9877288341522217, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5487179756164551, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7581300735473633, "metrics/particle/count/bias_mean": -0.18039216101169586, "metrics/particle/count/mae": 2.8784313201904297, "metrics/particle/count/prediction_mean": 69.81895446777344, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04374553635716438, "metrics/particle/event/energy_response_mean": 1.0141119956970215, "metrics/particle/event/met_abs_error": 3.8397817611694336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03455580025911331, "metrics/particle/event/scalar_pt_response_mean": 1.0146090984344482, "metrics/particle/event/vector_pt_closure": 0.034781526774168015, "metrics/particle/matched/delta_eta_abs_mean": 0.007785624824464321, "metrics/particle/matched/delta_phi_abs_mean": 0.008288566023111343, "metrics/particle/matched/delta_r_mean": 0.012821666896343231, "metrics/particle/matched/energy_relative_abs_mean": 0.0823609009385109, "metrics/particle/matched/pid_accuracy": 0.9728526473045349, "metrics/particle/matched/pt_relative_abs_mean": 0.07244893908500671, "metrics/particle/matching/duplicate_fraction": 0.026323918253183365, "metrics/particle/matching/efficiency": 0.8409415483474731, "metrics/particle/matching/f1": 0.8420265316963196, "metrics/particle/matching/purity": 0.8431143164634705, "metrics/particle/matching/target_coverage_dr0p05": 0.8240693211555481, "metrics/particle/matching/target_coverage_dr0p10": 0.8716794848442078, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.751753032207489, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8767028450965881, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.015094521455466747, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07798558473587036, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07140402495861053, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017629483714699745, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12646521627902985, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0032984530553221703, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.31149739027023315, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0008543205913156271, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.266116738319397, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8792.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.26138538122177124, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8061249852180481, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7984293699264526, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6941230297088623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11426124721765518, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 7.971259117126465, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.01160688791424036, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6535379886627197, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.02210625261068344, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.3996856212615967, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10511.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8179365396499634, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 1.0526231527328491, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.165081024169922, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2954983711242676, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.004603094421327114, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.12198439985513687, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.4110212326049805, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.2326456606388092, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.404610633850098, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.023098526522517204, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08614993095397949, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08450488746166229, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.023784130811691284, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13863719999790192, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01987447775900364, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9176015853881836, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.1042005643248558, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.133044958114624, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1912.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6786354780197144, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.358740210533142, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 57560.765625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 57560.62109375, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23698820173740387, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3089985.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07043719291687012, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.551192045211792, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.17786966264247894, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 8.020581245422363, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2882.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6223958134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.1479436457157135, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.518350601196289, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.0296508073806763, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.050597697496414185, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.1481439173221588, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.377325057983398, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 1.23165762424469, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.130992889404297, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010832265950739384, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08415669947862625, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07201483845710754, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.015908770263195038, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12846675515174866, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0004650441696867347, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.09925740957260132, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.007377068046480417, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5487712025642395, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19353.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1317903697490692, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7081627249717712, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.49899858236312866, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4066856801509857, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08006664365530014, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.190951824188232, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002221502363681793, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.22326046228408813, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.012179293669760227, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.571398138999939, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21607.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8815249800682068, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.9452658891677856, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.0244927406311035, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.20159912109375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.10774781554937363, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.17640052735805511, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3224637508392334, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.01562463492155075, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.524506092071533, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01285930722951889, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08247837424278259, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07263068854808807, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016913069412112236, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12855839729309082, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025285291485488415, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2133968025445938, "metrics/particle/momentum/all/accepted/pz_bias": 0.001628634287044406, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1345622539520264, "metrics/particle/momentum/all/accepted/samples": 30057.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21573838591575623, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7911525964736938, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4740.265625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4740.1650390625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10325714200735092, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 886685.1875, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010657142847776413, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5441675186157227, "metrics/particle/momentum/all/assigned/pz_bias": 0.02880391664803028, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.9241833686828613, "metrics/particle/momentum/all/assigned/samples": 35000.0, "metrics/particle/momentum/all/efficiency": 0.8401677012443542, "metrics/particle/momentum/all/event/energy_bias": 1.8499454259872437, "metrics/particle/momentum/all/event/p3_error_mean": 11.91356372833252, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.478882312774658, "metrics/particle/momentum/all/event/px_bias": 0.06175321340560913, "metrics/particle/momentum/all/event/py_bias": 0.09372778236865997, "metrics/particle/momentum/all/event/pz_abs_mean": 9.199827194213867, "metrics/particle/momentum/all/event/pz_bias": 1.448678731918335, "metrics/particle/momentum/all/event/pz_residual_rms": 17.87311363220215, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006602115463465452, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.037984754890203476, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.019923316314816475, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004994834773242474, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05854317545890808, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001338851056061685, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10764157772064209, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.0025797164998948574, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8541614413261414, "metrics/particle/momentum/class_1/accepted/samples": 16432.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03229490667581558, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.10135076940059662, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08350172638893127, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06019687280058861, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.01582503505051136, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.8126811981201172, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003556187730282545, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.19106413424015045, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01504679024219513, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.0338668823242188, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9738635420799255, "metrics/particle/momentum/class_1/event/energy_bias": 2.639552116394043, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.745734691619873, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.8319950103759766, "metrics/particle/momentum/class_1/event/px_bias": -0.42173832654953003, "metrics/particle/momentum/class_1/event/py_bias": -0.3694916069507599, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.962498188018799, "metrics/particle/momentum/class_1/event/pz_bias": 0.4721708595752716, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.758644104003906, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03488974645733833, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19240647554397583, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19313764572143555, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.019191093742847443, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23413878679275513, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030102789402008057, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.609886884689331, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.03993672505021095, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.772157907485962, "metrics/particle/momentum/class_2/accepted/samples": 1362.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8753380179405212, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.636865496635437, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 61144.74609375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 61144.29296875, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34279394149780273, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3184773.75, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06892738491296768, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.9382224082946777, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08873893320560455, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.125927448272705, "metrics/particle/momentum/class_2/assigned/samples": 2713.0, "metrics/particle/momentum/class_2/efficiency": 0.46232178807258606, "metrics/particle/momentum/class_2/event/energy_bias": -5.7097086906433105, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.187274932861328, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.387945652008057, "metrics/particle/momentum/class_2/event/px_bias": 0.4555068612098694, "metrics/particle/momentum/class_2/event/py_bias": -0.055879853665828705, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.906501770019531, "metrics/particle/momentum/class_2/event/pz_bias": 0.5200700163841248, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.717374801635742, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019234435632824898, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13386891782283783, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13383522629737854, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03828568384051323, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1755828857421875, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0011068539461120963, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.20054461061954498, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.005347361788153648, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7766700387001038, "metrics/particle/momentum/class_3/accepted/samples": 11745.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3089028000831604, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.593699336051941, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.4592033624649048, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.334241509437561, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1614551842212677, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 14.484862327575684, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008338376879692078, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.514850378036499, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.04188510775566101, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.435546875, "metrics/particle/momentum/class_3/assigned/samples": 14871.0, "metrics/particle/momentum/class_3/efficiency": 0.762068510055542, "metrics/particle/momentum/class_3/event/energy_bias": 5.778975486755371, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.605835914611816, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.487441062927246, "metrics/particle/momentum/class_3/event/px_bias": -0.15997891128063202, "metrics/particle/momentum/class_3/event/py_bias": 0.464545339345932, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.356359958648682, "metrics/particle/momentum/class_3/event/pz_bias": 0.4482799768447876, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.830928802490234, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.011204797774553299, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05282102897763252, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05333322659134865, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0038584291469305754, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10106723755598068, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.22210094332695007, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.04055478423833847, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7673418521881104, "metrics/particle/momentum/class_4/accepted/samples": 341.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.07830727100372314, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.5109920501708984, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3545291423797607, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.3015931844711304, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03591160103678703, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 20.26481819152832, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.28958219289779663, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0805726870894432, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9280421733856201, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9419889450073242, "metrics/particle/momentum/class_4/event/energy_bias": -0.6133729815483093, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2086503505706787, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.779360294342041, "metrics/particle/momentum/class_4/event/px_bias": 0.12671735882759094, "metrics/particle/momentum/class_4/event/py_bias": 0.03244534507393837, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7655736207962036, "metrics/particle/momentum/class_4/event/pz_bias": 0.14730972051620483, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.973738193511963, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004390768241137266, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.014276280999183655, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014383641071617603, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008125284686684608, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.03380680829286575, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12148412317037582, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.06795277446508408, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4507818818092346, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05017654597759247, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.031052764505147934, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030125418677926064, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01333539467304945, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11169448494911194, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.49264952540397644, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4405888020992279, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.192400932312012, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.2455001324415207, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8281168937683105, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5363850593566895, "metrics/particle/momentum/class_5/event/px_bias": 0.06124620512127876, "metrics/particle/momentum/class_5/event/py_bias": 0.022108538076281548, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5418757200241089, "metrics/particle/momentum/class_5/event/pz_bias": -0.13915181159973145, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.36985182762146, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7738237380981445, "step_valid_time": 19529.130907297134, "step_total_time": 19530.90473127365, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003545270702928, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05614892091940227, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3027, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2900, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2894, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9618573797678275, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580442682523951, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9599470374048328, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04195573174760489, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9598673300165838, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9560621076973902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9579609400860642, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9618573797678275, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0066475764920542, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06503579421219108, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2154, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2153, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9560585885486018, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9590382902938558, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9575461213603023, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04096170970614425, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9556147359076786, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9585930543187889, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.957101578128473, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9560585885486018, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008321536073139, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06268510543238981, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1200, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.933852140077821, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.933852140077821, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.933852140077821, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06614785992217899, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.933852140077821} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_46000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_46000.json new file mode 100644 index 0000000000000000000000000000000000000000..50a5853993e5b0a994164a3a97cb7b3734e5a3d4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_46000.json @@ -0,0 +1 @@ +{"epoch": 2.179268523782452, "train": {"Classification_binary": 2.342970848083496, "Classification": 0.01750798709690571, "Regression_pt": 0.11585962772369385, "Regression_eta": 0.009676704183220863, "Regression_sin_phi": 0.008584819734096527, "Regression_cos_phi": 0.007893032394349575, "Regression_energy": 0.11711946874856949, "Total": 7.931187629699707, "Total_unweighted": 2.619612487964332}, "train_group": {}, "valid": {"Classification_binary": 2.315119743347168, "Classification": 0.01822480435172717, "Regression_pt": 0.1165428360303243, "Regression_eta": 0.00981441761056582, "Regression_sin_phi": 0.0077726468443870544, "Regression_cos_phi": 0.007812963177760443, "Regression_energy": 0.11986905336380005, "Total": 7.936745325724284, "Total_unweighted": 2.595156464725733, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018721207976341248, "diagnostic/loss/clic_pf/Classification_binary": 2.319169044494629, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007348690647631884, "diagnostic/loss/clic_pf/Regression_energy": 0.12211860716342926, "diagnostic/loss/clic_pf/Regression_eta": 0.009091188199818134, "diagnostic/loss/clic_pf/Regression_pt": 0.11892108619213104, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007272796705365181, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02828342095017433, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18016153573989868, "diagnostic/regression/clic_pf/energy_residual_mean": 0.0438370406627655, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4074174165725708, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.007067635655403137, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1719145029783249, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06177689880132675, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4075237214565277, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9738803505897522, "metrics/particle/class_1/pid_accuracy": 0.988426923751831, "metrics/particle/class_2/efficiency": 0.46199819445610046, "metrics/particle/class_2/pid_accuracy": 0.7852087020874023, "metrics/particle/class_3/efficiency": 0.7657721638679504, "metrics/particle/class_3/pid_accuracy": 0.988516092300415, "metrics/particle/class_4/efficiency": 0.9457364082336426, "metrics/particle/class_4/pid_accuracy": 0.563524603843689, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.7515274882316589, "metrics/particle/count/bias_mean": -0.5032680034637451, "metrics/particle/count/mae": 2.9450979232788086, "metrics/particle/count/prediction_mean": 69.49607849121094, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.04257179796695709, "metrics/particle/event/energy_response_mean": 1.0041927099227905, "metrics/particle/event/met_abs_error": 3.560027837753296, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03222109004855156, "metrics/particle/event/scalar_pt_response_mean": 1.003238558769226, "metrics/particle/event/vector_pt_closure": 0.03344963490962982, "metrics/particle/matched/delta_eta_abs_mean": 0.007348007522523403, "metrics/particle/matched/delta_phi_abs_mean": 0.00831523071974516, "metrics/particle/matched/delta_r_mean": 0.012519020587205887, "metrics/particle/matched/energy_relative_abs_mean": 0.08261153101921082, "metrics/particle/matched/pid_accuracy": 0.9733251929283142, "metrics/particle/matched/pt_relative_abs_mean": 0.07193093001842499, "metrics/particle/matching/duplicate_fraction": 0.024433597922325134, "metrics/particle/matching/efficiency": 0.8411376476287842, "metrics/particle/matching/f1": 0.8441722989082336, "metrics/particle/matching/purity": 0.8472288846969604, "metrics/particle/matching/target_coverage_dr0p05": 0.8259554505348206, "metrics/particle/matching/target_coverage_dr0p10": 0.87156742811203, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7525187134742737, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8765441179275513, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014735816046595573, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07765936851501465, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0704125165939331, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.014023098163306713, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12522681057453156, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0031821797601878643, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.29734769463539124, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.00487592164427042, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1536266803741455, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8799.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25821295380592346, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8257858157157898, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.8296403288841248, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.72117018699646, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11209692806005478, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 12.33614730834961, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011448197066783905, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6471266746520996, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03127104416489601, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.7635817527770996, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10482.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8185877799987793, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.2621532082557678, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.035284519195557, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2520835399627686, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.12573868036270142, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.05804514139890671, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.306669235229492, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.03704322129487991, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.095117568969727, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022253278642892838, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08698058873414993, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08384931832551956, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.01893693208694458, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13819661736488342, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.0201446283608675, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8756847977638245, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.09506284445524216, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.91690993309021, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1936.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6757003664970398, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.0913020372390747, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.5661656856536865, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.415367364883423, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.22902889549732208, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 47.685997009277344, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06404455006122589, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.376507043838501, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.024679750204086304, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.5662946701049805, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2873.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6302083134651184, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.7705722451210022, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.022083759307861, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9632445573806763, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04998747631907463, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.108730249106884, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.881323337554932, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.4015021026134491, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.02353858947754, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010732841677963734, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08427353948354721, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07138092815876007, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014546927995979786, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12780025601387024, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006721820100210607, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1032862588763237, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006432197988033295, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6145207285881042, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19340.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1301381140947342, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.6343595385551453, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4455409646034241, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.35184481739997864, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08051948249340057, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 7.009102821350098, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002458255970850587, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21582114696502686, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.006674532778561115, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1462652683258057, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21560.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8809328675270081, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.8654188513755798, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.599640846252441, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 3.8687868118286133, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.24559135735034943, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.3389883041381836, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.1053414344787598, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.21106378734111786, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 3.8264033794403076, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012645583599805832, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08251269906759262, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07190021872520447, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.014676266349852085, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12775136530399323, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002660016529262066, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.209783673286438, "metrics/particle/momentum/all/accepted/pz_bias": 0.0005565987667068839, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0862375497817993, "metrics/particle/momentum/all/accepted/samples": 30075.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21347996592521667, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7294282913208008, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.7353500127792358, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.6325199007987976, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10221967846155167, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 16.22148895263672, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01022483129054308, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5230986475944519, "metrics/particle/momentum/all/assigned/pz_bias": 0.007297295145690441, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.7955238819122314, "metrics/particle/momentum/all/assigned/samples": 34915.0, "metrics/particle/momentum/all/efficiency": 0.8406708836555481, "metrics/particle/momentum/all/event/energy_bias": -1.898144245147705, "metrics/particle/momentum/all/event/p3_error_mean": 10.98225212097168, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.088661193847656, "metrics/particle/momentum/all/event/px_bias": -0.16984018683433533, "metrics/particle/momentum/all/event/py_bias": -0.2883031666278839, "metrics/particle/momentum/all/event/pz_abs_mean": 8.466772079467773, "metrics/particle/momentum/all/event/pz_bias": 0.15339505672454834, "metrics/particle/momentum/all/event/pz_residual_rms": 16.62677764892578, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0062778727151453495, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03769738972187042, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.018731294199824333, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0012774389469996095, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05564981326460838, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.001523739891126752, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10077239573001862, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.005308814812451601, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8237941861152649, "metrics/particle/momentum/class_1/accepted/samples": 16407.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03358004614710808, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09234890341758728, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07681789994239807, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.0494508258998394, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016477979719638824, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.8215004205703735, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003556398442015052, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.18316800892353058, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.017227355390787125, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.9965885877609253, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9723818898200989, "metrics/particle/momentum/class_1/event/energy_bias": 0.7805525064468384, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.717299461364746, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.9460182189941406, "metrics/particle/momentum/class_1/event/px_bias": -0.5139454007148743, "metrics/particle/momentum/class_1/event/py_bias": -0.24722546339035034, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.83969783782959, "metrics/particle/momentum/class_1/event/pz_bias": 0.5484752058982849, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.204730033874512, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03568350151181221, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1900777667760849, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19052600860595703, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03348517417907715, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2310248613357544, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032967034727334976, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6164100170135498, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.022509178146719933, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.599989652633667, "metrics/particle/momentum/class_2/accepted/samples": 1365.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8497487902641296, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6428685188293457, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.6828833818435669, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.2077300250530243, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.34502050280570984, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.262744426727295, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.070123091340065, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8942601680755615, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.09671188145875931, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.8218255043029785, "metrics/particle/momentum/class_2/assigned/samples": 2681.0, "metrics/particle/momentum/class_2/efficiency": 0.4633401334285736, "metrics/particle/momentum/class_2/event/energy_bias": -7.949038028717041, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.04857063293457, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.176697731018066, "metrics/particle/momentum/class_2/event/px_bias": 0.24743783473968506, "metrics/particle/momentum/class_2/event/py_bias": -0.28297242522239685, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.935525894165039, "metrics/particle/momentum/class_2/event/pz_bias": -0.6054888367652893, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.907817840576172, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.019048314541578293, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13447131216526031, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13373582065105438, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03947674110531807, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1757737398147583, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008486082660965621, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19991174340248108, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.008047529496252537, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7374688982963562, "metrics/particle/momentum/class_3/accepted/samples": 11784.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3084022104740143, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.4862886667251587, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5122253894805908, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.3885762691497803, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15844523906707764, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 24.8001708984375, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0072204601019620895, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4883055090904236, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.020562268793582916, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.284963369369507, "metrics/particle/momentum/class_3/assigned/samples": 14819.0, "metrics/particle/momentum/class_3/efficiency": 0.7645990252494812, "metrics/particle/momentum/class_3/event/energy_bias": 5.968463897705078, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.4303560256958, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.46577787399292, "metrics/particle/momentum/class_3/event/px_bias": -0.17357352375984192, "metrics/particle/momentum/class_3/event/py_bias": 0.20888546109199524, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.219904899597168, "metrics/particle/momentum/class_3/event/pz_bias": 0.32996654510498047, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.256294250488281, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010179152712225914, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.047904979437589645, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.04841584712266922, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0009555394062772393, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09048166871070862, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.20664381980895996, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.002431121189147234, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7478566765785217, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.0823211669921875, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.42951270937919617, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.36686891317367554, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.315696120262146, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.0234034061431885, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.24089615046977997, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.018315233290195465, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8283096551895142, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -0.37328916788101196, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2644692659378052, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8550267815589905, "metrics/particle/momentum/class_4/event/px_bias": 0.20615200698375702, "metrics/particle/momentum/class_4/event/py_bias": 0.013656881637871265, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7598631381988525, "metrics/particle/momentum/class_4/event/pz_bias": 0.06611743569374084, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.935255527496338, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.0037303294520825148, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.014795610681176186, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.01415649801492691, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.0034820574801415205, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.040401652455329895, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13016748428344727, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04003605246543884, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5872124433517456, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04909278452396393, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.031006600707769394, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030038394033908844, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.008940006606280804, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11807827651500702, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.4991646111011505, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4101935029029846, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.2058234214782715, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.3248330354690552, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7837961912155151, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5084534883499146, "metrics/particle/momentum/class_5/event/px_bias": 0.06408889591693878, "metrics/particle/momentum/class_5/event/py_bias": 0.019352350383996964, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5120011568069458, "metrics/particle/momentum/class_5/event/pz_bias": -0.18567529320716858, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.292100191116333, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7388103008270264, "step_valid_time": 19965.448502779007, "step_total_time": 19967.18731355667, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9985606971357797, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05413364654659614, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3030, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2906, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2901, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9638474295190713, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9590759075907591, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9614557485525227, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04092409240924092, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9621890547263682, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9574257425742574, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9598014888337469, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9638474295190713, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002648798184255, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06467351176786107, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2248, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2153, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2151, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9556147359076786, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9577402135231317, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9566762941568541, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04225978647686833, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9547270306258322, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9568505338078291, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557876027549433, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9556147359076786, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015772936332217, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06243528664495446, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1199, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9361867704280156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9354587869362364, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9358226371061844, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0645412130637636, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9330739299610895, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.932348367029549, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9327110073901206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9361867704280156} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_47000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_47000.json new file mode 100644 index 0000000000000000000000000000000000000000..84acfafe6e6e12e8f68c74e977358eaf29499923 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_47000.json @@ -0,0 +1 @@ +{"epoch": 2.226643926473375, "train": {"Classification_binary": 2.294009208679199, "Classification": 0.018589934334158897, "Regression_pt": 0.11442289501428604, "Regression_eta": 0.008452730253338814, "Regression_sin_phi": 0.006470759864896536, "Regression_cos_phi": 0.005415900144726038, "Regression_energy": 0.11583545058965683, "Total": 7.5421013832092285, "Total_unweighted": 2.5631968788802624}, "train_group": {}, "valid": {"Classification_binary": 2.3132068316141763, "Classification": 0.018195901066064835, "Regression_pt": 0.11667544643084209, "Regression_eta": 0.009805905322233835, "Regression_sin_phi": 0.0077639296650886536, "Regression_cos_phi": 0.0077833204219738645, "Regression_energy": 0.12011249860127766, "Total": 7.936089197794597, "Total_unweighted": 2.5935438331216574, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.01865498349070549, "diagnostic/loss/clic_pf/Classification_binary": 2.318431854248047, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007371420972049236, "diagnostic/loss/clic_pf/Regression_energy": 0.1225300058722496, "diagnostic/loss/clic_pf/Regression_eta": 0.009075420908629894, "diagnostic/loss/clic_pf/Regression_pt": 0.11909814178943634, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007285464089363813, "diagnostic/regression/clic_pf/energy_pred_mean": 0.028878454118967056, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1808909922838211, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04443207010626793, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41005805134773254, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.005841512233018875, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17309576272964478, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06300301849842072, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4104413390159607, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9741785526275635, "metrics/particle/class_1/pid_accuracy": 0.988471269607544, "metrics/particle/class_2/efficiency": 0.4663960337638855, "metrics/particle/class_2/pid_accuracy": 0.7903771996498108, "metrics/particle/class_3/efficiency": 0.7652764320373535, "metrics/particle/class_3/pid_accuracy": 0.9876636862754822, "metrics/particle/class_4/efficiency": 0.9437984228134155, "metrics/particle/class_4/pid_accuracy": 0.5626283288002014, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7459349632263184, "metrics/particle/count/bias_mean": -0.5784313678741455, "metrics/particle/count/mae": 2.9758169651031494, "metrics/particle/count/prediction_mean": 69.42091369628906, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.0423511378467083, "metrics/particle/event/energy_response_mean": 1.004330039024353, "metrics/particle/event/met_abs_error": 3.90100359916687, "metrics/particle/event/scalar_pt_relative_abs_error": 0.032662197947502136, "metrics/particle/event/scalar_pt_response_mean": 1.0054759979248047, "metrics/particle/event/vector_pt_closure": 0.03446359187364578, "metrics/particle/matched/delta_eta_abs_mean": 0.006836528889834881, "metrics/particle/matched/delta_phi_abs_mean": 0.008217702619731426, "metrics/particle/matched/delta_r_mean": 0.012041575275361538, "metrics/particle/matched/energy_relative_abs_mean": 0.08206105977296829, "metrics/particle/matched/pid_accuracy": 0.9731343388557434, "metrics/particle/matched/pt_relative_abs_mean": 0.07185325026512146, "metrics/particle/matching/duplicate_fraction": 0.02434707246720791, "metrics/particle/matching/efficiency": 0.8414177298545837, "metrics/particle/matching/f1": 0.8449086546897888, "metrics/particle/matching/purity": 0.8484286665916443, "metrics/particle/matching/target_coverage_dr0p05": 0.825787365436554, "metrics/particle/matching/target_coverage_dr0p10": 0.8716514706611633, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7533683776855469, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8768149018287659, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.01438137050718069, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07832200825214386, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07141241431236267, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.016148757189512253, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12671014666557312, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028451120015233755, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3030277192592621, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0023730499669909477, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1853219270706177, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8787.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.2495001256465912, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.8084896206855774, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7541016936302185, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6487360596656799, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11198165267705917, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.462522506713867, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011561245657503605, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6595163345336914, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.029038600623607635, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.829124689102173, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10466.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8174713850021362, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.007081904448568821, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 4.837758541107178, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.244527578353882, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.22141240537166595, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.024609385058283806, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.069738388061523, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.3046182692050934, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 7.993075847625732, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022308839485049248, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08794107288122177, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08427761495113373, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.022942593321204185, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1399243175983429, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021331945434212685, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.935188353061676, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.12693865597248077, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.2816050052642822, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1922.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6855258941650391, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1723507642745972, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.599095344543457, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.4572863578796387, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2265734225511551, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 41.45011901855469, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06398601084947586, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3463573455810547, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04870175942778587, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.812285423278809, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2860.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6256510615348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.666374921798706, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.870102882385254, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9199790954589844, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.046940676867961884, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.030094731599092484, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.746399402618408, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.329058974981308, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.799346923828125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010039769113063812, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08308503776788712, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0707925409078598, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.011107579804956913, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1264648735523224, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005679178284481168, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1011228933930397, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.004923171363770962, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5923885107040405, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19369.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12882408499717712, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.554989218711853, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4047263562679291, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.30815640091896057, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.0794459655880928, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 4.4460906982421875, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0022235605865716934, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2116362601518631, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0005789175629615784, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.1115455627441406, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21587.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8822538256645203, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.11118588596582413, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 5.075989723205566, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.238096237182617, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.2005583494901657, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.20358632504940033, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.3671650886535645, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.040633805096149445, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.677645683288574, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012092126533389091, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08200386166572571, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07183533161878586, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.013336575590074062, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12743869423866272, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025600106455385685, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21340465545654297, "metrics/particle/momentum/all/accepted/pz_bias": 0.004247862845659256, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1509017944335938, "metrics/particle/momentum/all/accepted/samples": 30078.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21060344576835632, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6815549731254578, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6892179846763611, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5863054990768433, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10125168412923813, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.409259796142578, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010082203894853592, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5207708477973938, "metrics/particle/momentum/all/assigned/pz_bias": 0.004357513040304184, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.6391091346740723, "metrics/particle/momentum/all/assigned/samples": 34913.0, "metrics/particle/momentum/all/efficiency": 0.8407546877861023, "metrics/particle/momentum/all/event/energy_bias": -0.7704789638519287, "metrics/particle/momentum/all/event/p3_error_mean": 11.343840599060059, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.4172210693359375, "metrics/particle/momentum/all/event/px_bias": -0.02608664333820343, "metrics/particle/momentum/all/event/py_bias": -0.1981009840965271, "metrics/particle/momentum/all/event/pz_abs_mean": 8.608009338378906, "metrics/particle/momentum/all/event/pz_bias": -0.6743110418319702, "metrics/particle/momentum/all/event/pz_residual_rms": 15.062339782714844, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006250426173210144, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.037367600947618484, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01907971315085888, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0033809214364737272, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05698763579130173, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014612762024626136, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11028923839330673, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.009591891430318356, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.9924996495246887, "metrics/particle/momentum/class_1/accepted/samples": 16424.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.03327228128910065, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.08676277101039886, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.07041601836681366, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.04547327384352684, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.0152916070073843, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.6568156480789185, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.002963489852845669, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.1717660129070282, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.0021391892805695534, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.3069957494735718, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9733894467353821, "metrics/particle/momentum/class_1/event/energy_bias": 1.8216619491577148, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.304605007171631, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.854401111602783, "metrics/particle/momentum/class_1/event/px_bias": -0.3879361152648926, "metrics/particle/momentum/class_1/event/py_bias": -0.32411932945251465, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.449077606201172, "metrics/particle/momentum/class_1/event/pz_bias": 0.2368677854537964, "metrics/particle/momentum/class_1/event/pz_residual_rms": 10.977924346923828, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03454693406820297, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19016429781913757, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1917162984609604, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.024861624464392662, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2307986319065094, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031976744532585144, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5779619216918945, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.02895413525402546, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.583791732788086, "metrics/particle/momentum/class_2/accepted/samples": 1376.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.836113691329956, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6365367770195007, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7062785029411316, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.24695301055908203, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3363095223903656, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.3758537769317627, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.0721726194024086, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8947677612304688, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.03849112242460251, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.017097473144531, "metrics/particle/momentum/class_2/assigned/samples": 2688.0, "metrics/particle/momentum/class_2/efficiency": 0.4670740067958832, "metrics/particle/momentum/class_2/event/energy_bias": -7.125555515289307, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.905261993408203, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.321956634521484, "metrics/particle/momentum/class_2/event/px_bias": 0.2977595031261444, "metrics/particle/momentum/class_2/event/py_bias": -0.3088970482349396, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.613231658935547, "metrics/particle/momentum/class_2/event/pz_bias": -0.616180956363678, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.241423606872559, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017791280522942543, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13359737396240234, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1329457014799118, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03206847980618477, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1745823174715042, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0007652410422451794, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19901546835899353, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.005536062642931938, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.6995621919631958, "metrics/particle/momentum/class_3/accepted/samples": 11761.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30388379096984863, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3605417013168335, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.386085033416748, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.2568050622940063, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1590249240398407, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.258546829223633, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007157809566706419, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.49383464455604553, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.020530307665467262, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.302013635635376, "metrics/particle/momentum/class_3/assigned/samples": 14809.0, "metrics/particle/momentum/class_3/efficiency": 0.763106644153595, "metrics/particle/momentum/class_3/event/energy_bias": 5.290133476257324, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.381132125854492, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.418069362640381, "metrics/particle/momentum/class_3/event/px_bias": -0.1477544754743576, "metrics/particle/momentum/class_3/event/py_bias": 0.3663717210292816, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.160480499267578, "metrics/particle/momentum/class_3/event/pz_bias": -0.28380483388900757, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.169814109802246, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010458787903189659, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05013798922300339, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05050547420978546, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0035622683353722095, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09390180557966232, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.21529459953308105, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.00433193938806653, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7499651908874512, "metrics/particle/momentum/class_4/accepted/samples": 340.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.09597814083099365, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.2879403829574585, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.2261850833892822, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.1716864109039307, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 19.553958892822266, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.27114740014076233, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.02736094780266285, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.8861401081085205, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.939226508140564, "metrics/particle/momentum/class_4/event/energy_bias": -0.4769522547721863, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.1219868659973145, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.7137342095375061, "metrics/particle/momentum/class_4/event/px_bias": 0.13753154873847961, "metrics/particle/momentum/class_4/event/py_bias": 0.04181218147277832, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.717185378074646, "metrics/particle/momentum/class_4/event/pz_bias": 0.1318737417459488, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.842223644256592, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004034803248941898, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.0160172451287508, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015536594204604626, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008193649351596832, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04687170311808586, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12598012387752533, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.03375225514173508, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.48917677998542786, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04942630976438522, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.031636543571949005, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03143493831157684, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.01358242891728878, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.1202184408903122, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5008159875869751, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4100169837474823, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.210635662078857, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.27976566553115845, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.8575592637062073, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5448290109634399, "metrics/particle/momentum/class_5/event/px_bias": 0.07431292533874512, "metrics/particle/momentum/class_5/event/py_bias": 0.026731479912996292, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.5681469440460205, "metrics/particle/momentum/class_5/event/pz_bias": -0.14306676387786865, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.419156074523926, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7051153182983398, "step_valid_time": 20403.445395469666, "step_total_time": 20405.1505112648, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995693693309481, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05267107234098156, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2896, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625207296849088, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9602911978821972, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9614046711943018, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03970880211780278, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960530679933665, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9583057577763071, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9594169289382144, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625207296849088, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010455492539747, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06304653434168261, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2135, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2132, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9476253883710608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9526996876394467, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9501557632398754, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04730031236055333, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9462938304482912, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9513609995537706, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9488206497552292, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9476253883710608, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0033157758113678, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06143578369620928, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1206, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1203, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9385214007782101, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392523364485982, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9388867263526665, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06074766355140187, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9361867704280156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9369158878504673, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9365511872323862, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9385214007782101} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_48000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_48000.json new file mode 100644 index 0000000000000000000000000000000000000000..554a049d8a960b8e2617063186d9155bdc04971c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_48000.json @@ -0,0 +1 @@ +{"epoch": 2.274019329164298, "train": {"Classification_binary": 2.3280844688415527, "Classification": 0.018618104979395866, "Regression_pt": 0.10500752180814743, "Regression_eta": 0.008087089285254478, "Regression_sin_phi": 0.007441469468176365, "Regression_cos_phi": 0.007723065558820963, "Regression_energy": 0.10733797401189804, "Total": 7.509589672088623, "Total_unweighted": 2.582299693953246}, "train_group": {}, "valid": {"Classification_binary": 2.3070178031921387, "Classification": 0.018052836259206135, "Regression_pt": 0.11691675583521526, "Regression_eta": 0.009791556745767593, "Regression_sin_phi": 0.007784734790523847, "Regression_cos_phi": 0.0077591271450122195, "Regression_energy": 0.12019868691762288, "Total": 7.927595774332683, "Total_unweighted": 2.5875215008854866, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018433988094329834, "diagnostic/loss/clic_pf/Classification_binary": 2.3161590099334717, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007307859603315592, "diagnostic/loss/clic_pf/Regression_energy": 0.1228010281920433, "diagnostic/loss/clic_pf/Regression_eta": 0.008981864899396896, "diagnostic/loss/clic_pf/Regression_pt": 0.11955408751964569, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007314956746995449, "diagnostic/regression/clic_pf/energy_pred_mean": 0.024525605142116547, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.1786186695098877, "diagnostic/regression/clic_pf/energy_residual_mean": 0.040079232305288315, "diagnostic/regression/clic_pf/energy_residual_rms": 0.40740767121315, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.009893038310110569, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.17111587524414062, "diagnostic/regression/clic_pf/pt_residual_mean": 0.058951497077941895, "diagnostic/regression/clic_pf/pt_residual_rms": 0.40788397192955017, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9738207459449768, "metrics/particle/class_1/pid_accuracy": 0.9884670376777649, "metrics/particle/class_2/efficiency": 0.47158321738243103, "metrics/particle/class_2/pid_accuracy": 0.7917264699935913, "metrics/particle/class_3/efficiency": 0.7690483927726746, "metrics/particle/class_3/pid_accuracy": 0.987500011920929, "metrics/particle/class_4/efficiency": 0.9457364082336426, "metrics/particle/class_4/pid_accuracy": 0.5512295365333557, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.7484787106513977, "metrics/particle/count/bias_mean": -0.3535947799682617, "metrics/particle/count/mae": 2.9653594493865967, "metrics/particle/count/prediction_mean": 69.645751953125, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.042062725871801376, "metrics/particle/event/energy_response_mean": 1.00783371925354, "metrics/particle/event/met_abs_error": 3.730865478515625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031957536935806274, "metrics/particle/event/scalar_pt_response_mean": 1.0072845220565796, "metrics/particle/event/vector_pt_closure": 0.03364221751689911, "metrics/particle/matched/delta_eta_abs_mean": 0.006742486730217934, "metrics/particle/matched/delta_phi_abs_mean": 0.008452226407825947, "metrics/particle/matched/delta_r_mean": 0.01215753611177206, "metrics/particle/matched/energy_relative_abs_mean": 0.0819099023938179, "metrics/particle/matched/pid_accuracy": 0.972940981388092, "metrics/particle/matched/pt_relative_abs_mean": 0.07156216353178024, "metrics/particle/matching/duplicate_fraction": 0.025244466960430145, "metrics/particle/matching/efficiency": 0.8433412313461304, "metrics/particle/matching/f1": 0.8454766273498535, "metrics/particle/matching/purity": 0.8476228713989258, "metrics/particle/matching/target_coverage_dr0p05": 0.8278601765632629, "metrics/particle/matching/target_coverage_dr0p10": 0.8732481002807617, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7562442421913147, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8784675598144531, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.0144929438829422, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07800810039043427, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07071677595376968, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01087277103215456, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12476049363613129, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0028351100627332926, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3009546101093292, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.006992179900407791, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.142116904258728, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8818.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25192689895629883, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7378547787666321, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7182509303092957, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6078858971595764, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.10905447602272034, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.667339324951172, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.010972235351800919, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6294891834259033, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.031176479533314705, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.3224902153015137, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10481.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8203553557395935, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.3532522916793823, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.105480194091797, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2653768062591553, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3790779411792755, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.05446848273277283, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.388698101043701, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.027692951261997223, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.134001731872559, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022647880017757416, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08627866208553314, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08332041651010513, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019003843888640404, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.13593247532844543, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.020123839378356934, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8851115703582764, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.10748319327831268, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.8846306800842285, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1938.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6680098176002502, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.494452953338623, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 21934.94140625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 21934.796875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23623692989349365, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 1174938.625, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06655052304267883, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4725871086120605, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.1401340663433075, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.841362476348877, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2870.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.630859375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.4929218292236328, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.084505558013916, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9925447702407837, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.03614049404859543, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.09154766798019409, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.928959369659424, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.761601984500885, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.138765335083008, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010181380435824394, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08334904909133911, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07105781137943268, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.013090066611766815, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12694688141345978, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006697233766317368, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10114241391420364, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003993234597146511, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5816434621810913, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19411.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12705488502979279, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.650122344493866, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.45431849360466003, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3594425618648529, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07827779650688171, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.6831207275390625, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0024099736474454403, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21620076894760132, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.002716176211833954, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3906068801879883, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21577.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8841668963432312, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.3892449140548706, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.814847946166992, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.030227184295654, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.09795200824737549, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.22460123896598816, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.239020347595215, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.002474423497915268, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.073276519775391, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012242554686963558, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08197606354951859, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07174590229988098, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01282185222953558, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.1269109696149826, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0025524580851197243, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.20991280674934387, "metrics/particle/momentum/all/accepted/pz_bias": 0.0022916672751307487, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.064681053161621, "metrics/particle/momentum/all/accepted/samples": 30167.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20897553861141205, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7458264231681824, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1802.869140625, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1802.765625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1004924401640892, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 336797.8125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01024965662509203, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.525623083114624, "metrics/particle/momentum/all/assigned/pz_bias": 0.022547880187630653, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.8045542240142822, "metrics/particle/momentum/all/assigned/samples": 34928.0, "metrics/particle/momentum/all/efficiency": 0.8432424664497375, "metrics/particle/momentum/all/event/energy_bias": -0.5289144515991211, "metrics/particle/momentum/all/event/p3_error_mean": 11.43699836730957, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.26690149307251, "metrics/particle/momentum/all/event/px_bias": 0.440889447927475, "metrics/particle/momentum/all/event/py_bias": -0.1875220686197281, "metrics/particle/momentum/all/event/pz_abs_mean": 8.841217994689941, "metrics/particle/momentum/all/event/pz_bias": 0.7314346432685852, "metrics/particle/momentum/all/event/pz_residual_rms": 17.017229080200195, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006195650901645422, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.037460412830114365, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01914142817258835, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.003959814552217722, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05681535229086876, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014013282489031553, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10235645622015, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0014900545356795192, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7896552681922913, "metrics/particle/momentum/class_1/accepted/samples": 16413.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.032171059399843216, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09728331118822098, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08524306863546371, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.06059759110212326, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016181614249944687, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9654059410095215, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003556398442015052, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.19003726541996002, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.005172977223992348, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.037184715270996, "metrics/particle/momentum/class_1/assigned/samples": 16871.0, "metrics/particle/momentum/class_1/efficiency": 0.9727374911308289, "metrics/particle/momentum/class_1/event/energy_bias": 1.4091942310333252, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.661281108856201, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.8634657859802246, "metrics/particle/momentum/class_1/event/px_bias": -0.2663300633430481, "metrics/particle/momentum/class_1/event/py_bias": -0.2273242026567459, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.816370487213135, "metrics/particle/momentum/class_1/event/pz_bias": 0.35762593150138855, "metrics/particle/momentum/class_1/event/pz_residual_rms": 13.67132568359375, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.03502906113862991, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.1911064237356186, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1929498314857483, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.031133104115724564, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2326706498861313, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03136136755347252, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5770610570907593, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.05580609664320946, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5525197982788086, "metrics/particle/momentum/class_2/accepted/samples": 1403.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8311665058135986, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6396452188491821, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 23365.298828125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 23364.8359375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3314773440361023, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 1212711.0, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.07015590369701385, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.8770904541015625, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.10227924585342407, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.968077659606934, "metrics/particle/momentum/class_2/assigned/samples": 2694.0, "metrics/particle/momentum/class_2/efficiency": 0.4762389659881592, "metrics/particle/momentum/class_2/event/energy_bias": -6.381293773651123, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.986618041992188, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.264409065246582, "metrics/particle/momentum/class_2/event/px_bias": 0.4328629672527313, "metrics/particle/momentum/class_2/event/py_bias": -0.40706077218055725, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.816680908203125, "metrics/particle/momentum/class_2/event/pz_bias": 0.38544341921806335, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.627608299255371, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.018103571608662605, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13270863890647888, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1318100392818451, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03067210502922535, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17291103303432465, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008453085320070386, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.1981896460056305, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.001285974052734673, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.713722825050354, "metrics/particle/momentum/class_3/accepted/samples": 11830.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.30290788412094116, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.5213878154754639, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.5488234758377075, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.4179105758666992, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.15709562599658966, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 21.226152420043945, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0072204601019620895, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.48708677291870117, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.03523371368646622, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1793551445007324, "metrics/particle/momentum/class_3/assigned/samples": 14819.0, "metrics/particle/momentum/class_3/efficiency": 0.7675837278366089, "metrics/particle/momentum/class_3/event/energy_bias": 5.311936378479004, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.409123420715332, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.373282432556152, "metrics/particle/momentum/class_3/event/px_bias": 0.09251043200492859, "metrics/particle/momentum/class_3/event/py_bias": 0.36258164048194885, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.2341156005859375, "metrics/particle/momentum/class_3/event/pz_bias": 0.19523514807224274, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.050214767456055, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010373756289482117, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05030864477157593, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05082206800580025, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.003553323447704315, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09244164079427719, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.20904582738876343, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.021597428247332573, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7076416611671448, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.05355414003133774, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.3729865252971649, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.2962847650051117, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.24602238833904266, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03867403417825699, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 2.133237600326538, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.24702851474285126, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.042957890778779984, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.7821040153503418, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.5205789804458618, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2114953994750977, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8223913311958313, "metrics/particle/momentum/class_4/event/px_bias": 0.12861843407154083, "metrics/particle/momentum/class_4/event/py_bias": 0.05330737680196762, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7268040180206299, "metrics/particle/momentum/class_4/event/pz_bias": 0.013446161523461342, "metrics/particle/momentum/class_4/event/pz_residual_rms": 2.65765118598938, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004284712020307779, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.015803750604391098, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.015376192517578602, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.007945663295686245, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.04248715937137604, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.13235792517662048, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.061789289116859436, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.5652730464935303, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04944133758544922, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.02904483489692211, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.02753441222012043, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.00955752469599247, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11188305914402008, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5186817646026611, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4496641159057617, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.258153438568115, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.3481719493865967, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7631403207778931, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.5021154284477234, "metrics/particle/momentum/class_5/event/px_bias": 0.05322767794132233, "metrics/particle/momentum/class_5/event/py_bias": 0.030973903834819794, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4934682846069336, "metrics/particle/momentum/class_5/event/pz_bias": -0.2203160673379898, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2667436599731445, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 2.0079684257507324, "step_valid_time": 20838.2025513649, "step_total_time": 20840.210520267487, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003489285249443, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.054211213724190466, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2898, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9625207296849088, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9596560846560847, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9610862725616824, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04034391534391534, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9611940298507463, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9583333333333334, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9597615499254843, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9625207296849088, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015411462148323, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.062149406199827784, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2245, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2148, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2144, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9533954727030626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956792873051225, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9550911516229436, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04320712694877506, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9516200621393698, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.955011135857461, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9533125833703868, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9533954727030626, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004068474630825, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06175054592176499, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1202, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377431906614786, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362859362859363, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9370139968895801, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06371406371406371, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.935408560311284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.933954933954934, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9346811819595645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377431906614786} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_49000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_49000.json new file mode 100644 index 0000000000000000000000000000000000000000..0e61e340d53c17d8ff72dd05db23f0f0f1cd7785 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_49000.json @@ -0,0 +1 @@ +{"epoch": 2.3213947318552206, "train": {"Classification_binary": 2.243170738220215, "Classification": 0.01690404862165451, "Regression_pt": 0.11712438613176346, "Regression_eta": 0.007473597303032875, "Regression_sin_phi": 0.006081203930079937, "Regression_cos_phi": 0.00640155840665102, "Regression_energy": 0.11864321678876877, "Total": 7.495535373687744, "Total_unweighted": 2.5157987494021654}, "train_group": {}, "valid": {"Classification_binary": 2.3043999671936035, "Classification": 0.018114008009433746, "Regression_pt": 0.11602165301640828, "Regression_eta": 0.009782971193393072, "Regression_sin_phi": 0.007793979719281197, "Regression_cos_phi": 0.007786520446340243, "Regression_energy": 0.11937063932418823, "Total": 7.904557545979817, "Total_unweighted": 2.5832697389026484, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018528543412685394, "diagnostic/loss/clic_pf/Classification_binary": 2.3099217414855957, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007428121753036976, "diagnostic/loss/clic_pf/Regression_energy": 0.1214752048254013, "diagnostic/loss/clic_pf/Regression_eta": 0.009042608551681042, "diagnostic/loss/clic_pf/Regression_pt": 0.11828839778900146, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007334995083510876, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03328794613480568, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18095967173576355, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04884156957268715, "diagnostic/regression/clic_pf/energy_residual_rms": 0.41076672077178955, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0006539811147376895, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1736287772655487, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0681905522942543, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4118911027908325, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.974297821521759, "metrics/particle/class_1/pid_accuracy": 0.9881054162979126, "metrics/particle/class_2/efficiency": 0.46684709191322327, "metrics/particle/class_2/pid_accuracy": 0.7927536368370056, "metrics/particle/class_3/efficiency": 0.7593275308609009, "metrics/particle/class_3/pid_accuracy": 0.988106369972229, "metrics/particle/class_4/efficiency": 0.944767415523529, "metrics/particle/class_4/pid_accuracy": 0.5815384387969971, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.7530612349510193, "metrics/particle/count/bias_mean": -0.6287581920623779, "metrics/particle/count/mae": 2.9869279861450195, "metrics/particle/count/prediction_mean": 69.37059020996094, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.044206321239471436, "metrics/particle/event/energy_response_mean": 1.0083839893341064, "metrics/particle/event/met_abs_error": 3.8600194454193115, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03335145115852356, "metrics/particle/event/scalar_pt_response_mean": 1.006928563117981, "metrics/particle/event/vector_pt_closure": 0.03470192849636078, "metrics/particle/matched/delta_eta_abs_mean": 0.006732287351042032, "metrics/particle/matched/delta_phi_abs_mean": 0.008230067789554596, "metrics/particle/matched/delta_r_mean": 0.011964789591729641, "metrics/particle/matched/energy_relative_abs_mean": 0.08185216039419174, "metrics/particle/matched/pid_accuracy": 0.9734106659889221, "metrics/particle/matched/pt_relative_abs_mean": 0.07173267006874084, "metrics/particle/matching/duplicate_fraction": 0.024845248088240623, "metrics/particle/matching/efficiency": 0.8389247059822083, "metrics/particle/matching/f1": 0.842709481716156, "metrics/particle/matching/purity": 0.8465285301208496, "metrics/particle/matching/target_coverage_dr0p05": 0.8249096870422363, "metrics/particle/matching/target_coverage_dr0p10": 0.8704189658164978, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7520145177841187, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8750501871109009, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014105251990258694, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07789424061775208, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07100822776556015, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01472822017967701, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12624695897102356, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003194159362465143, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.30528372526168823, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.0045565273612737656, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2187718152999878, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8766.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.25444477796554565, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.7253145575523376, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.6584194898605347, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.5497197508811951, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11015408486127853, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 5.3621392250061035, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.011580055579543114, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6403768062591553, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03708912804722786, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.3997879028320312, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10449.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8155177235603333, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.299330472946167, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 4.973874092102051, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.1954686641693115, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.28643691539764404, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.1098710149526596, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.280688285827637, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.0173904150724411, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.017462730407715, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022082490846514702, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08567879348993301, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0828065574169159, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.019118988886475563, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1362372636795044, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.021232521161437035, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.9006136059761047, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.05789892375469208, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 3.0567078590393066, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1931.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.663528561592102, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.3014212846755981, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.7587151527404785, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 2.6206820011138916, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2239382266998291, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 42.134403228759766, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06318006664514542, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.4015889167785645, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.12571701407432556, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 7.802945613861084, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2849.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6285807490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.44706210494041443, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 6.201061725616455, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.9195847511291504, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.0227239690721035, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12455054372549057, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 6.078258037567139, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.6632022857666016, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.44418716430664, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01004859060049057, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08321064710617065, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07105138897895813, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.016790520399808884, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.1270775943994522, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006219872739166021, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10058629512786865, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.003385838819667697, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5670174360275269, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19293.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.12858004868030548, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.608931303024292, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4262427091598511, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.3362210988998413, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08146223425865173, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 5.074763774871826, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0023659307043999434, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2154010534286499, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.0018162379274144769, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.3609846830368042, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21556.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8787920475006104, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.11251895129680634, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.80588436126709, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.03117561340332, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.12178199738264084, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.031157296150922775, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.238434314727783, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.17493364214897156, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.206815719604492, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012009182944893837, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08181559294462204, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07179565727710724, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.016337640583515167, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12744608521461487, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027009001933038235, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21193109452724457, "metrics/particle/momentum/all/accepted/pz_bias": 0.00021798317902721465, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.114728331565857, "metrics/particle/momentum/all/accepted/samples": 29990.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.21004067361354828, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.7004270553588867, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6865063309669495, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5869606733322144, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.1017099916934967, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 13.025463104248047, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.010099271312355995, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5215071439743042, "metrics/particle/momentum/all/assigned/pz_bias": 0.022518590092658997, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.801598310470581, "metrics/particle/momentum/all/assigned/samples": 34854.0, "metrics/particle/momentum/all/efficiency": 0.8382949233055115, "metrics/particle/momentum/all/event/energy_bias": -0.8589115142822266, "metrics/particle/momentum/all/event/p3_error_mean": 11.65926742553711, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.332030773162842, "metrics/particle/momentum/all/event/px_bias": 0.14193095266819, "metrics/particle/momentum/all/event/py_bias": 0.26557886600494385, "metrics/particle/momentum/all/event/pz_abs_mean": 9.066006660461426, "metrics/particle/momentum/all/event/pz_bias": 0.4708782434463501, "metrics/particle/momentum/all/event/pz_residual_rms": 17.36418914794922, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.0061382208950817585, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03721999377012253, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01935749128460884, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004265292081981897, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.05728498846292496, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0017050298629328609, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10445957630872726, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.0017787599936127663, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8516813516616821, "metrics/particle/momentum/class_1/accepted/samples": 16422.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.032384827733039856, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09902777522802353, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.08059608191251755, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.056612640619277954, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016180654987692833, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.8550498485565186, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.003556187730282545, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.18870878219604492, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.01756754331290722, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.038242816925049, "metrics/particle/momentum/class_1/assigned/samples": 16872.0, "metrics/particle/momentum/class_1/efficiency": 0.9732708930969238, "metrics/particle/momentum/class_1/event/energy_bias": 1.6649365425109863, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.818068504333496, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.890420913696289, "metrics/particle/momentum/class_1/event/px_bias": -0.4911015033721924, "metrics/particle/momentum/class_1/event/py_bias": -0.28934136033058167, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.980066776275635, "metrics/particle/momentum/class_1/event/pz_bias": 0.46767833828926086, "metrics/particle/momentum/class_1/event/pz_residual_rms": 14.074443817138672, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.0347227044403553, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19277939200401306, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19426199793815613, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.03205833211541176, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2338259518146515, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.030368763953447342, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.6034960746765137, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.031333327293395996, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.6414825916290283, "metrics/particle/momentum/class_2/accepted/samples": 1383.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8321825265884399, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6253163814544678, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7493393421173096, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.28677698969841003, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3308355212211609, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.1525728702545166, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06931435316801071, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.860578775405884, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.08097650855779648, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 7.009016513824463, "metrics/particle/momentum/class_2/assigned/samples": 2669.0, "metrics/particle/momentum/class_2/efficiency": 0.46945011615753174, "metrics/particle/momentum/class_2/event/energy_bias": -6.738876819610596, "metrics/particle/momentum/class_2/event/p3_error_mean": 11.767895698547363, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.043757438659668, "metrics/particle/momentum/class_2/event/px_bias": 0.4957302510738373, "metrics/particle/momentum/class_2/event/py_bias": 0.045636169612407684, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.627403259277344, "metrics/particle/momentum/class_2/event/pz_bias": -0.029614461585879326, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.224688529968262, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017757587134838104, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13347014784812927, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13266785442829132, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03963650017976761, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17431776225566864, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.000942830229178071, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.19975502789020538, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.004461466334760189, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7617435455322266, "metrics/particle/momentum/class_3/accepted/samples": 11667.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3058372437953949, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3902884721755981, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3591525554656982, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.236695408821106, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1604035496711731, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.59037208557129, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.007109486032277346, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.48565155267715454, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.024048108607530594, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.1587817668914795, "metrics/particle/momentum/class_3/assigned/samples": 14769.0, "metrics/particle/momentum/class_3/efficiency": 0.7570075392723083, "metrics/particle/momentum/class_3/event/energy_bias": 5.034912109375, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.37031364440918, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.407279968261719, "metrics/particle/momentum/class_3/event/px_bias": -0.003073647618293762, "metrics/particle/momentum/class_3/event/py_bias": 0.41492828726768494, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.178223133087158, "metrics/particle/momentum/class_3/event/pz_bias": 0.09156759828329086, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.944252014160156, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01018055435270071, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.047043249011039734, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.0476565882563591, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0027856307569891214, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09102220833301544, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.19485005736351013, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.010995082557201385, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7074629068374634, "metrics/particle/momentum/class_4/accepted/samples": 342.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08388705551624298, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.4557384252548218, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.3344428539276123, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.2841134071350098, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.041436463594436646, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 21.070772171020508, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2418595254421234, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.022716790437698364, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.838509738445282, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9447513818740845, "metrics/particle/momentum/class_4/event/energy_bias": -0.43121829628944397, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2269587516784668, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.7967497110366821, "metrics/particle/momentum/class_4/event/px_bias": 0.07213225215673447, "metrics/particle/momentum/class_4/event/py_bias": 0.07641799747943878, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7706204652786255, "metrics/particle/momentum/class_4/event/pz_bias": 0.129801407456398, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.0768117904663086, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.003821048652753234, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.014341655187308788, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014002957381308079, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.004921452607959509, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.037913814187049866, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.1452401578426361, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.05792746692895889, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.6491129398345947, "metrics/particle/momentum/class_5/accepted/samples": 176.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.032903604209423065, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.0701645165681839, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.06206599995493889, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.042567748576402664, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.41342291235923767, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5367573499679565, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.40992313623428345, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.240464687347412, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9670329689979553, "metrics/particle/momentum/class_5/event/energy_bias": -0.38866502046585083, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7558145523071289, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.48921525478363037, "metrics/particle/momentum/class_5/event/px_bias": 0.0682436004281044, "metrics/particle/momentum/class_5/event/py_bias": 0.017937740311026573, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.4926895797252655, "metrics/particle/momentum/class_5/event/pz_bias": -0.18855467438697815, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2791919708251953, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.7613766193389893, "step_valid_time": 21271.433800458908, "step_total_time": 21273.195177555084, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005811702758338, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05465124118929021, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3022, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2893, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9608623548922056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9586366644606221, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9597482193142289, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0413633355393779, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9595356550580432, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.957313037723362, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9584230578101706, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9608623548922056, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013398375179579, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06241478298292169, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2240, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2150, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2149, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.954283177984909, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9598214285714286, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9570442911195193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04017857142857143, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9538393253439857, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.959375, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9565991542399288, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.954283177984909, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002512933295951, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06293325600601962, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1287, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1205, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1201, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9377431906614786, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362859362859363, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9370139968895801, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06371406371406371, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9346303501945525, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9331779331779332, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9339035769828927, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9377431906614786} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_5000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_5000.json new file mode 100644 index 0000000000000000000000000000000000000000..9eca90c91c90377e308b7c84b475a302c35f286a --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_5000.json @@ -0,0 +1 @@ +{"epoch": 0.23687701345461437, "train": {"Classification_binary": 3.1711699962615967, "Classification": 0.02571527287364006, "Regression_pt": 0.16414527595043182, "Regression_eta": 0.010758662596344948, "Regression_sin_phi": 0.009830825962126255, "Regression_cos_phi": 0.010037797503173351, "Regression_energy": 0.16331085562705994, "Total": 10.735814094543457, "Total_unweighted": 3.554968686774373}, "train_group": {}, "valid": {"Classification_binary": 3.073399861653646, "Classification": 0.025722478826840717, "Regression_pt": 0.16429721315701803, "Regression_eta": 0.01227028047045072, "Regression_sin_phi": 0.009546330819527308, "Regression_cos_phi": 0.009520487238963446, "Regression_energy": 0.16858492294947305, "Total": 10.737082163492838, "Total_unweighted": 3.463341575115919, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02651137113571167, "diagnostic/loss/clic_pf/Classification_binary": 3.0560543537139893, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.009139979258179665, "diagnostic/loss/clic_pf/Regression_energy": 0.17239989340305328, "diagnostic/loss/clic_pf/Regression_eta": 0.011229684576392174, "diagnostic/loss/clic_pf/Regression_pt": 0.16894486546516418, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.009276803582906723, "diagnostic/regression/clic_pf/energy_pred_mean": 0.02577102743089199, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.21080243587493896, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04132464900612831, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4684849977493286, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0032134749926626682, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20396243035793304, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06563106179237366, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4730961322784424, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9614964127540588, "metrics/particle/class_1/pid_accuracy": 0.9882571697235107, "metrics/particle/class_2/efficiency": 0.3459630012512207, "metrics/particle/class_2/pid_accuracy": 0.6368970274925232, "metrics/particle/class_3/efficiency": 0.7043647170066833, "metrics/particle/class_3/pid_accuracy": 0.9798953533172607, "metrics/particle/class_4/efficiency": 0.9360465407371521, "metrics/particle/class_4/pid_accuracy": 0.19358177483081818, "metrics/particle/class_5/efficiency": 0.98591548204422, "metrics/particle/class_5/pid_accuracy": 0.5163265466690063, "metrics/particle/count/bias_mean": -1.2039215564727783, "metrics/particle/count/mae": 3.7986927032470703, "metrics/particle/count/prediction_mean": 68.79542541503906, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.07049459964036942, "metrics/particle/event/energy_response_mean": 0.9925950169563293, "metrics/particle/event/met_abs_error": 8.600305557250977, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05721907317638397, "metrics/particle/event/scalar_pt_response_mean": 1.0006808042526245, "metrics/particle/event/vector_pt_closure": 0.06229640543460846, "metrics/particle/matched/delta_eta_abs_mean": 0.012887866236269474, "metrics/particle/matched/delta_phi_abs_mean": 0.014427931047976017, "metrics/particle/matched/delta_r_mean": 0.02151964046061039, "metrics/particle/matched/energy_relative_abs_mean": 0.08924110233783722, "metrics/particle/matched/pid_accuracy": 0.9607937335968018, "metrics/particle/matched/pt_relative_abs_mean": 0.07709481567144394, "metrics/particle/matching/duplicate_fraction": 0.03178885951638222, "metrics/particle/matching/efficiency": 0.7990083694458008, "metrics/particle/matching/f1": 0.8059390783309937, "metrics/particle/matching/purity": 0.8129910826683044, "metrics/particle/matching/target_coverage_dr0p05": 0.7291011214256287, "metrics/particle/matching/target_coverage_dr0p10": 0.8322393298149109, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7123782634735107, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8430237174034119, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023602716624736786, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08349065482616425, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07396689802408218, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.006493479013442993, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13004037737846375, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.004033736884593964, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.32903704047203064, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.023210017010569572, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2754082679748535, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8181.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33613258600234985, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9739264249801636, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.004256010055542, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.8680477142333984, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1416504830121994, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 9.75249195098877, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017766989767551422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8808576464653015, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.04241674765944481, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.490903615951538, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10300.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7610940337181091, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.0083632469177246, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 10.073643684387207, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 4.674924373626709, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.1908954232931137, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.03382384777069092, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 8.566417694091797, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -2.5317416191101074, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.302005767822266, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.030092766508460045, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08929362893104553, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08292347937822342, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.000744238612242043, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1417573243379593, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.019370460882782936, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8578184843063354, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.00019754681852646172, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9817018508911133, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1652.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8048007488250732, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9634273052215576, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 59771.6640625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 59771.47265625, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2841029167175293, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 3183484.75, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08530137687921524, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.4022250175476074, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.057749561965465546, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 12.411873817443848, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2837.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5377604365348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -1.1797688007354736, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.21807861328125, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.402576208114624, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04626722261309624, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": -0.17335659265518188, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.008907318115234, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.1865832805633545, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 20.907989501953125, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01990792341530323, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.09154912829399109, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07771271467208862, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008194182068109512, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13294637203216553, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006946671055629849, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12237963080406189, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03462238609790802, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6082593202590942, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18714.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.17024321854114532, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 1.1570998430252075, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.9192437529563904, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.8050470352172852, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09801998734474182, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 42.98456573486328, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003408984746783972, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.34277886152267456, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.0010665347799658775, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 7.084352493286133, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21414.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8524187207221985, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.11728896200656891, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 12.757189750671387, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 10.066741943359375, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -2.386636734008789, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -3.5374205112457275, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 7.141423225402832, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 0.5075199007987976, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 44.38608932495117, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021556168794631958, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08910920470952988, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07694078236818314, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.00727566983550787, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13264912366867065, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0027323360554873943, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2241629809141159, "metrics/particle/momentum/all/accepted/pz_bias": -0.02933681756258011, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.1059843301773071, "metrics/particle/momentum/all/accepted/samples": 28547.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.27180036902427673, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.1687020063400269, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 4908.75, "metrics/particle/momentum/all/assigned/pt_relative_bias": 4908.62353515625, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12630604207515717, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 912225.375, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.01441347599029541, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.7543981671333313, "metrics/particle/momentum/all/assigned/pz_bias": -0.008564028888940811, "metrics/particle/momentum/all/assigned/pz_residual_rms": 6.883888244628906, "metrics/particle/momentum/all/assigned/samples": 34551.0, "metrics/particle/momentum/all/efficiency": 0.7979594469070435, "metrics/particle/momentum/all/event/energy_bias": -3.3054211139678955, "metrics/particle/momentum/all/event/p3_error_mean": 21.829301834106445, "metrics/particle/momentum/all/event/pt_vector_error_mean": 11.642741203308105, "metrics/particle/momentum/all/event/px_bias": -2.242008686065674, "metrics/particle/momentum/all/event/py_bias": -3.6769533157348633, "metrics/particle/momentum/all/event/pz_abs_mean": 16.315536499023438, "metrics/particle/momentum/all/event/pz_bias": -1.837638258934021, "metrics/particle/momentum/all/event/pz_residual_rms": 48.8707389831543, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.010640686377882957, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0470307283103466, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02547665685415268, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.007290222216397524, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06251702457666397, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0012969367671757936, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.1199292242527008, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.018490321934223175, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8181493878364563, "metrics/particle/momentum/class_1/accepted/samples": 16192.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0585222989320755, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.5634242296218872, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.5362061262130737, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.48957034945487976, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.025010371580719948, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 46.82931900024414, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005452497862279415, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3426835536956787, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.035886477679014206, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 8.082174301147461, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9596396684646606, "metrics/particle/momentum/class_1/event/energy_bias": 7.658690452575684, "metrics/particle/momentum/class_1/event/p3_error_mean": 14.793481826782227, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 8.490250587463379, "metrics/particle/momentum/class_1/event/px_bias": -2.629603862762451, "metrics/particle/momentum/class_1/event/py_bias": -0.7003753781318665, "metrics/particle/momentum/class_1/event/pz_abs_mean": 10.685521125793457, "metrics/particle/momentum/class_1/event/pz_bias": 1.265999674797058, "metrics/particle/momentum/class_1/event/pz_residual_rms": 46.53722381591797, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04546666145324707, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.21041136980056763, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2108214795589447, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.057897042483091354, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2501208782196045, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.04236453026533127, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.8530911207199097, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.05613842234015465, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 4.070515155792236, "metrics/particle/momentum/class_2/accepted/samples": 1015.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0576980113983154, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8717907071113586, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 67128.3515625, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 67127.7734375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.43269991874694824, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3373772.25, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.09857482463121414, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.8611958026885986, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.23417486250400543, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.480795860290527, "metrics/particle/momentum/class_2/assigned/samples": 2526.0, "metrics/particle/momentum/class_2/efficiency": 0.3445349633693695, "metrics/particle/momentum/class_2/event/energy_bias": -12.976078987121582, "metrics/particle/momentum/class_2/event/p3_error_mean": 18.139385223388672, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 9.19061279296875, "metrics/particle/momentum/class_2/event/px_bias": 0.08213440328836441, "metrics/particle/momentum/class_2/event/py_bias": -2.3827574253082275, "metrics/particle/momentum/class_2/event/pz_abs_mean": 13.615870475769043, "metrics/particle/momentum/class_2/event/pz_bias": -1.4870805740356445, "metrics/particle/momentum/class_2/event/pz_residual_rms": 20.960617065429688, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.036053650081157684, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14308562874794006, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.14321428537368774, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.0360737107694149, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18537774682044983, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0012934220721945167, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.22613906860351562, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.04149461165070534, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7975795269012451, "metrics/particle/momentum/class_3/accepted/samples": 10824.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3894633948802948, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.947718858718872, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.8344165086746216, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.691123604774475, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.19366100430488586, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 20.013715744018555, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.01067908015102148, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.7042022347450256, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.013514187186956406, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.5652971267700195, "metrics/particle/momentum/class_3/assigned/samples": 14608.0, "metrics/particle/momentum/class_3/efficiency": 0.7023099064826965, "metrics/particle/momentum/class_3/event/energy_bias": 5.127889156341553, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.691166877746582, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.954897880554199, "metrics/particle/momentum/class_3/event/px_bias": 0.2672004699707031, "metrics/particle/momentum/class_3/event/py_bias": -0.6620657444000244, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.76949405670166, "metrics/particle/momentum/class_3/event/pz_bias": -1.5860681533813477, "metrics/particle/momentum/class_3/event/pz_residual_rms": 16.361783981323242, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01496503222733736, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.04418399557471275, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.044599466025829315, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.012641909532248974, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09106740355491638, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.23246438801288605, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.005213371478021145, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.873379111289978, "metrics/particle/momentum/class_4/accepted/samples": 337.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10844147205352783, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.5848328471183777, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.43873974680900574, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3704526424407959, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049723755568265915, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.3809828758239746, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.36964234709739685, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0832078754901886, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.2931870222091675, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9309391975402832, "metrics/particle/momentum/class_4/event/energy_bias": -1.8712618350982666, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1364173889160156, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.4313565492630005, "metrics/particle/momentum/class_4/event/px_bias": -0.09182848781347275, "metrics/particle/momentum/class_4/event/py_bias": 0.20205897092819214, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.296762466430664, "metrics/particle/momentum/class_4/event/pz_bias": 0.33786752820014954, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.129878044128418, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.00912522617727518, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028283286839723587, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02650945633649826, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.009464846923947334, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06529105454683304, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.281170129776001, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.18838851153850555, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8191899657249451, "metrics/particle/momentum/class_5/accepted/samples": 179.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.0178250540047884, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03865455836057663, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.036580294370651245, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.012119361199438572, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.11096428334712982, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5985457897186279, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.45245110988616943, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.069553375244141, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9835164546966553, "metrics/particle/momentum/class_5/event/energy_bias": -1.2446590662002563, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.65867018699646, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 1.130276083946228, "metrics/particle/momentum/class_5/event/px_bias": 0.13008898496627808, "metrics/particle/momentum/class_5/event/py_bias": -0.13381372392177582, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.9872763156890869, "metrics/particle/momentum/class_5/event/pz_bias": -0.3683566153049469, "metrics/particle/momentum/class_5/event/pz_residual_rms": 4.530978679656982, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4018352031707764, "step_valid_time": 2171.3709313869476, "step_total_time": 2172.772766828537, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9898862371458267, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08720242019542801, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3006, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2842, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2829, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9426202321724709, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9454424484364604, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9440292310247467, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.054557551563539586, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9383084577114428, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9411177644710579, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9397110114598903, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9426202321724709, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9895213677431102, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09977856930332374, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2227, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2074, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2062, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9205503772747448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9312977099236641, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9258928571428572, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06870229007633588, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9152241455836663, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9259092950157162, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9205357142857142, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9205503772747448, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9934289938961465, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1007181989018533, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1273, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1135, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1127, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8832684824902723, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8915946582875098, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.887412040656763, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10840534171249018, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8770428015564202, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8853102906520032, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8811571540265832, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8832684824902723} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_50000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..cea22c90ad6815c04afea4bbbef66a85d80ee4d4 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 2.3687701345461436, "train": {"Classification_binary": 2.187082290649414, "Classification": 0.01576068438589573, "Regression_pt": 0.11640752106904984, "Regression_eta": 0.007098764181137085, "Regression_sin_phi": 0.007383483462035656, "Regression_cos_phi": 0.007542036939412355, "Regression_energy": 0.11530418694019318, "Total": 7.46658992767334, "Total_unweighted": 2.456578967627138}, "train_group": {}, "valid": {"Classification_binary": 2.3063138326009116, "Classification": 0.01811784381667773, "Regression_pt": 0.11603219310442607, "Regression_eta": 0.009697478885451952, "Regression_sin_phi": 0.007708355784416199, "Regression_cos_phi": 0.007719564562042554, "Regression_energy": 0.11935381094614665, "Total": 7.893003463745117, "Total_unweighted": 2.5849430797000728, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.018562931567430496, "diagnostic/loss/clic_pf/Classification_binary": 2.315140724182129, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.007291653659194708, "diagnostic/loss/clic_pf/Regression_energy": 0.12250511348247528, "diagnostic/loss/clic_pf/Regression_eta": 0.00906357541680336, "diagnostic/loss/clic_pf/Regression_pt": 0.11909979581832886, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.007303342688828707, "diagnostic/regression/clic_pf/energy_pred_mean": 0.029332367703318596, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.18062613904476166, "diagnostic/regression/clic_pf/energy_residual_mean": 0.044885993003845215, "diagnostic/regression/clic_pf/energy_residual_rms": 0.40801531076431274, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.004525783006101847, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1730719953775406, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06431876122951508, "diagnostic/regression/clic_pf/pt_residual_rms": 0.40895530581474304, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9738803505897522, "metrics/particle/class_1/pid_accuracy": 0.9872022271156311, "metrics/particle/class_2/efficiency": 0.46763643622398376, "metrics/particle/class_2/pid_accuracy": 0.7940679788589478, "metrics/particle/class_3/efficiency": 0.7622588872909546, "metrics/particle/class_3/pid_accuracy": 0.9872755408287048, "metrics/particle/class_4/efficiency": 0.9515503644943237, "metrics/particle/class_4/pid_accuracy": 0.5885946750640869, "metrics/particle/class_5/efficiency": 0.9899396300315857, "metrics/particle/class_5/pid_accuracy": 0.7642276287078857, "metrics/particle/count/bias_mean": -0.8163398504257202, "metrics/particle/count/mae": 3.054248332977295, "metrics/particle/count/prediction_mean": 69.1830062866211, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.043610796332359314, "metrics/particle/event/energy_response_mean": 1.0064804553985596, "metrics/particle/event/met_abs_error": 3.8686211109161377, "metrics/particle/event/scalar_pt_relative_abs_error": 0.033539872616529465, "metrics/particle/event/scalar_pt_response_mean": 1.0060985088348389, "metrics/particle/event/vector_pt_closure": 0.03439237177371979, "metrics/particle/matched/delta_eta_abs_mean": 0.006895925849676132, "metrics/particle/matched/delta_phi_abs_mean": 0.008190654218196869, "metrics/particle/matched/delta_r_mean": 0.012064119800925255, "metrics/particle/matched/energy_relative_abs_mean": 0.08242029696702957, "metrics/particle/matched/pid_accuracy": 0.9727603197097778, "metrics/particle/matched/pt_relative_abs_mean": 0.0721035972237587, "metrics/particle/matching/duplicate_fraction": 0.02474256046116352, "metrics/particle/matching/efficiency": 0.8401479125022888, "metrics/particle/matching/f1": 0.8450756072998047, "metrics/particle/matching/purity": 0.8500614166259766, "metrics/particle/matching/target_coverage_dr0p05": 0.8257219791412354, "metrics/particle/matching/target_coverage_dr0p10": 0.8705310225486755, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7514635920524597, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8748821020126343, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.014132675714790821, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07875197380781174, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0716778039932251, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.01511546690016985, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1269281804561615, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002850302029401064, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.308758407831192, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.006580663379281759, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.1937474012374878, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8771.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.24963749945163727, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.778779923915863, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 0.7707167267799377, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 0.6642534136772156, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.11069615930318832, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 11.039470672607422, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.009958824142813683, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.6318135857582092, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.039238642901182175, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 2.5787031650543213, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10443.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.8159828782081604, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.3919921815395355, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 5.077181816101074, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 2.2558815479278564, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.34144702553749084, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.0719829797744751, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 4.36331844329834, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -0.08065137267112732, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 8.824131965637207, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.022002428770065308, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08636333793401718, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08340872079133987, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.02263563498854637, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1378551572561264, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.022337662056088448, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8982915878295898, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": 0.11199020594358444, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.9892101287841797, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1925.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.6665937900543213, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.1943268775939941, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 2.0564992427825928, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 1.9131829738616943, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.23041798174381256, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 14.976997375488281, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.06533192843198776, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 2.3866915702819824, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": 0.131217360496521, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 6.838697910308838, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2847.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.6266276240348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.30389395356178284, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 5.6305742263793945, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 0.8998817205429077, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.04066616669297218, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.08141724765300751, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 5.504866600036621, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": 0.3217211961746216, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 13.051036834716797, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.010267919860780239, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08373355865478516, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.071331687271595, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.014265062287449837, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.12707991898059845, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005693876300938427, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.10084634274244308, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.006723829545080662, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5757492780685425, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19319.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1272563487291336, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.5547841191291809, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.4309164583683014, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.33725646138191223, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.07937392592430115, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.160326957702637, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.002322233049198985, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.21677519381046295, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.012141026556491852, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.2150977849960327, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21531.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8799763321876526, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": -0.26416268944740295, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 4.87406587600708, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 4.0646281242370605, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.039957694709300995, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.21407881379127502, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 2.2994308471679688, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.13673430681228638, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 4.514920711517334, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.012149869464337826, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08244649320840836, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07220738381147385, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01505041029304266, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.12775418162345886, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002632017247378826, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.21274636685848236, "metrics/particle/momentum/all/accepted/pz_bias": 0.004777694586664438, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0967456102371216, "metrics/particle/momentum/all/accepted/samples": 30015.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.20805585384368896, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.6742512583732605, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.6657335758209229, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.5641739368438721, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.10111714154481888, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 8.851824760437012, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.009764222428202629, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.5186619758605957, "metrics/particle/momentum/all/assigned/pz_bias": 0.01498913113027811, "metrics/particle/momentum/all/assigned/pz_residual_rms": 2.594419002532959, "metrics/particle/momentum/all/assigned/samples": 34821.0, "metrics/particle/momentum/all/efficiency": 0.8389937281608582, "metrics/particle/momentum/all/event/energy_bias": -0.17606444656848907, "metrics/particle/momentum/all/event/p3_error_mean": 11.288646697998047, "metrics/particle/momentum/all/event/pt_vector_error_mean": 5.310091972351074, "metrics/particle/momentum/all/event/px_bias": 0.260823130607605, "metrics/particle/momentum/all/event/py_bias": -0.20464454591274261, "metrics/particle/momentum/all/event/pz_abs_mean": 8.721515655517578, "metrics/particle/momentum/all/event/pz_bias": 0.1043354868888855, "metrics/particle/momentum/all/event/pz_residual_rms": 14.861047744750977, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.006256045773625374, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03770057484507561, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.01929132454097271, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.0037543203216046095, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.057306766510009766, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0014626119518652558, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.10752725601196289, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.004562686197459698, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8625190258026123, "metrics/particle/momentum/class_1/accepted/samples": 16409.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0348614864051342, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.09902609139680862, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.0852741226553917, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.060670457780361176, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.016713092103600502, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.9250493049621582, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.0030818467494100332, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.1780880093574524, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.008451271802186966, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 1.269731879234314, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9725004434585571, "metrics/particle/momentum/class_1/event/energy_bias": 1.9608783721923828, "metrics/particle/momentum/class_1/event/p3_error_mean": 7.591118812561035, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 3.980294704437256, "metrics/particle/momentum/class_1/event/px_bias": -0.36555221676826477, "metrics/particle/momentum/class_1/event/py_bias": -0.2378382384777069, "metrics/particle/momentum/class_1/event/pz_abs_mean": 5.693149566650391, "metrics/particle/momentum/class_1/event/pz_bias": -0.08082716166973114, "metrics/particle/momentum/class_1/event/pz_residual_rms": 11.464877128601074, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.034574270248413086, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19217714667320251, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.1939823478460312, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.026797359809279442, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23350515961647034, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031204644590616226, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.5764411687850952, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.04755951836705208, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.5244557857513428, "metrics/particle/momentum/class_2/accepted/samples": 1378.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 0.8191945552825928, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.6457962393760681, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.7300658822059631, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.2655797600746155, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.33470919728279114, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.0793657302856445, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.06904315203428268, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 2.872148275375366, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.017301423475146294, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 6.949925422668457, "metrics/particle/momentum/class_2/assigned/samples": 2665.0, "metrics/particle/momentum/class_2/efficiency": 0.4677528738975525, "metrics/particle/momentum/class_2/event/energy_bias": -6.385705471038818, "metrics/particle/momentum/class_2/event/p3_error_mean": 12.036866188049316, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 6.180669784545898, "metrics/particle/momentum/class_2/event/px_bias": 0.4805542230606079, "metrics/particle/momentum/class_2/event/py_bias": -0.4862007796764374, "metrics/particle/momentum/class_2/event/pz_abs_mean": 8.8851957321167, "metrics/particle/momentum/class_2/event/pz_bias": 0.00021488219499588013, "metrics/particle/momentum/class_2/event/pz_residual_rms": 14.48279857635498, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.017944198101758957, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.13418783247470856, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1335229128599167, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03617729991674423, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1747630089521408, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010250277118757367, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2010210156440735, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.0011979457922279835, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7478156685829163, "metrics/particle/momentum/class_3/accepted/samples": 11707.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3007078766822815, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.3165333271026611, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.3015336990356445, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.1752430200576782, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1580839902162552, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 12.775765419006348, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.0069204154424369335, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.4878920614719391, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.031419798731803894, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 2.248788356781006, "metrics/particle/momentum/class_3/assigned/samples": 14739.0, "metrics/particle/momentum/class_3/efficiency": 0.7596029043197632, "metrics/particle/momentum/class_3/event/energy_bias": 4.9496564865112305, "metrics/particle/momentum/class_3/event/p3_error_mean": 8.387679100036621, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 4.488210678100586, "metrics/particle/momentum/class_3/event/px_bias": -0.08625999093055725, "metrics/particle/momentum/class_3/event/py_bias": 0.4243929386138916, "metrics/particle/momentum/class_3/event/pz_abs_mean": 6.156322002410889, "metrics/particle/momentum/class_3/event/pz_bias": 0.32137495279312134, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.498470306396484, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.010185504332184792, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05105913430452347, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05147802084684372, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.006091881077736616, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09493963420391083, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2136412411928177, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.009527183137834072, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7519406080245972, "metrics/particle/momentum/class_4/accepted/samples": 343.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08562169969081879, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 1.8509548902511597, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 1.6676139831542969, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.6166943311691284, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.03867403417825699, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 25.471282958984375, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.2956390976905823, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.0022633036132901907, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9922353625297546, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9475138187408447, "metrics/particle/momentum/class_4/event/energy_bias": -0.4044240117073059, "metrics/particle/momentum/class_4/event/p3_error_mean": 1.2257717847824097, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 0.8057349920272827, "metrics/particle/momentum/class_4/event/px_bias": 0.15284983813762665, "metrics/particle/momentum/class_4/event/py_bias": 0.09795703738927841, "metrics/particle/momentum/class_4/event/pz_abs_mean": 0.7598601579666138, "metrics/particle/momentum/class_4/event/pz_bias": 0.09005753695964813, "metrics/particle/momentum/class_4/event/pz_residual_rms": 3.029273509979248, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.004568477626889944, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.01535108219832182, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.014808768406510353, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.008106923662126064, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.034142784774303436, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.12471509724855423, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.04359644278883934, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.4633978009223938, "metrics/particle/momentum/class_5/accepted/samples": 178.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.05611535906791687, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.06460913270711899, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.055342528969049454, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.03564629331231117, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.41289716958999634, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5665400624275208, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.2023659646511078, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.4239044189453125, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9780219793319702, "metrics/particle/momentum/class_5/event/energy_bias": -0.2964698076248169, "metrics/particle/momentum/class_5/event/p3_error_mean": 0.7563563585281372, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.48751604557037354, "metrics/particle/momentum/class_5/event/px_bias": 0.07923132181167603, "metrics/particle/momentum/class_5/event/py_bias": -0.002955499105155468, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.49982067942619324, "metrics/particle/momentum/class_5/event/pz_bias": -0.22648471593856812, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.2854015827178955, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.8204336166381836, "step_valid_time": 21708.37345647812, "step_total_time": 21710.193890333176, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002600500878442, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05410891333675405, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3020, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2901, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2897, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9621890547263682, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9605960264900663, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9613918806959404, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03940397350993378, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9608623548922056, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9592715231788079, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.96006628003314, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9621890547263682, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023021267075738, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06462489322523313, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2241, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2147, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2145, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9529516200621394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9580544399821508, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9554962171784601, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04194556001784917, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.952063914780293, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9571619812583668, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9546061415220294, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9529516200621394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028742461586038, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06274769297000704, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1284, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1198, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1195, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.932295719844358, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9330218068535826, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9326586220319191, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06697819314641744, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299610894941635, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9306853582554517, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9303230829116388, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.932295719844358} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_6000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_6000.json new file mode 100644 index 0000000000000000000000000000000000000000..96b44e7e55499883e85e2b0eec611deea344ff92 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_6000.json @@ -0,0 +1 @@ +{"epoch": 0.28425241614553726, "train": {"Classification_binary": 2.9773805141448975, "Classification": 0.026174621656537056, "Regression_pt": 0.1645021140575409, "Regression_eta": 0.008721078746020794, "Regression_sin_phi": 0.00904948078095913, "Regression_cos_phi": 0.010255733504891396, "Regression_energy": 0.16597765684127808, "Total": 10.473355293273926, "Total_unweighted": 3.362061199732125}, "train_group": {}, "valid": {"Classification_binary": 2.96041202545166, "Classification": 0.02432333678007126, "Regression_pt": 0.15497086445490518, "Regression_eta": 0.01155689979592959, "Regression_sin_phi": 0.00921375056107839, "Regression_cos_phi": 0.009302246073881784, "Regression_energy": 0.15840659538904825, "Total": 10.215822219848633, "Total_unweighted": 3.3281857185065746, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02488504722714424, "diagnostic/loss/clic_pf/Classification_binary": 2.945887565612793, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008865566924214363, "diagnostic/loss/clic_pf/Regression_energy": 0.1612706035375595, "diagnostic/loss/clic_pf/Regression_eta": 0.010768960230052471, "diagnostic/loss/clic_pf/Regression_pt": 0.15860483050346375, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008906753733754158, "diagnostic/regression/clic_pf/energy_pred_mean": 0.043831389397382736, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2062326967716217, "diagnostic/regression/clic_pf/energy_residual_mean": 0.059385016560554504, "diagnostic/regression/clic_pf/energy_residual_rms": 0.46458542346954346, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.012624796479940414, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20238469541072845, "diagnostic/regression/clic_pf/pt_residual_mean": 0.0814693346619606, "diagnostic/regression/clic_pf/pt_residual_rms": 0.4698669910430908, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.955552875995636, "metrics/particle/class_1/pid_accuracy": 0.9872063994407654, "metrics/particle/class_2/efficiency": 0.3752819001674652, "metrics/particle/class_2/pid_accuracy": 0.659254789352417, "metrics/particle/class_3/efficiency": 0.7290871739387512, "metrics/particle/class_3/pid_accuracy": 0.983060359954834, "metrics/particle/class_4/efficiency": 0.9273256063461304, "metrics/particle/class_4/pid_accuracy": 0.2967607080936432, "metrics/particle/class_5/efficiency": 0.9839034080505371, "metrics/particle/class_5/pid_accuracy": 0.6134969592094421, "metrics/particle/count/bias_mean": -0.6692810654640198, "metrics/particle/count/mae": 3.635294198989868, "metrics/particle/count/prediction_mean": 69.33006286621094, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.058807533234357834, "metrics/particle/event/energy_response_mean": 0.9962929487228394, "metrics/particle/event/met_abs_error": 6.627245903015137, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04714960232377052, "metrics/particle/event/scalar_pt_response_mean": 1.003103494644165, "metrics/particle/event/vector_pt_closure": 0.05095398426055908, "metrics/particle/matched/delta_eta_abs_mean": 0.012802894227206707, "metrics/particle/matched/delta_phi_abs_mean": 0.01205479260534048, "metrics/particle/matched/delta_r_mean": 0.01969323866069317, "metrics/particle/matched/energy_relative_abs_mean": 0.08553685992956161, "metrics/particle/matched/pid_accuracy": 0.9632633924484253, "metrics/particle/matched/pt_relative_abs_mean": 0.07701189070940018, "metrics/particle/matching/duplicate_fraction": 0.03165684640407562, "metrics/particle/matching/efficiency": 0.8092606067657471, "metrics/particle/matching/f1": 0.8131479620933533, "metrics/particle/matching/purity": 0.8170728087425232, "metrics/particle/matching/target_coverage_dr0p05": 0.7690547704696655, "metrics/particle/matching/target_coverage_dr0p10": 0.8444523215293884, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.71745765209198, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8485419750213623, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.022048169746994972, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08304939419031143, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07587245851755142, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.017134176567196846, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.13259486854076385, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.002879769541323185, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3449688255786896, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.0047273701056838036, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.214540958404541, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8334.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.33749765157699585, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 0.9967438578605652, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2236640453338623, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0965113639831543, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1428019255399704, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 19.41301155090332, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.016135266050696373, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8570831418037415, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.08219823986291885, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.0422871112823486, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10350.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7753279209136963, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.9923681020736694, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.792191028594971, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.543609857559204, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.36086344718933105, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.7056276202201843, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.677955150604248, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 0.8819069266319275, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 11.7218599319458, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03230347856879234, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09158238023519516, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08647359907627106, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.015000296756625175, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.1422453373670578, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016036655753850937, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8529399633407593, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.018840238451957703, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.817000150680542, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1746.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7790209054946899, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.5290600061416626, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 12422.0595703125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 12421.8876953125, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.26612040400505066, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 657996.8125, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.07695048302412033, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2067151069641113, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13714174926280975, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.610285758972168, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2807.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.568359375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -2.30659556388855, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.400555610656738, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2692596912384033, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.09913782775402069, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.12000174820423126, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.235687255859375, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -1.3141945600509644, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.679845809936523, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01776786707341671, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0860072523355484, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07689657807350159, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.018837295472621918, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13307903707027435, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0007432181155309081, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12384890764951706, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.000598015496507287, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.6392320990562439, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18837.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1644473522901535, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7125691771507263, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5714641809463501, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.46852821111679077, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09637994319200516, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 9.315044403076172, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.003159998217597604, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2891663908958435, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.010714186355471611, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.254481554031372, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21519.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8580213189125061, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.07969459891319275, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.245270729064941, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.801509380340576, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": 0.5571873188018799, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 1.1872667074203491, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.02047872543335, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.206071138381958, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 13.277503967285156, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.01987912505865097, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08549141883850098, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.0771796777844429, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.01811477541923523, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13351157307624817, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002282394329085946, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2315988838672638, "metrics/particle/momentum/all/accepted/pz_bias": -0.00016467759269289672, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0818759202957153, "metrics/particle/momentum/all/accepted/samples": 28917.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2658482789993286, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.8634833693504333, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1006.2775268554688, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1006.1619262695312, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.12397623807191849, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 187210.625, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013006113469600677, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6948503851890564, "metrics/particle/momentum/all/assigned/pz_bias": 0.020081710070371628, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.102269172668457, "metrics/particle/momentum/all/assigned/samples": 34676.0, "metrics/particle/momentum/all/efficiency": 0.8083018660545349, "metrics/particle/momentum/all/event/energy_bias": -3.219269037246704, "metrics/particle/momentum/all/event/p3_error_mean": 17.897911071777344, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.852770805358887, "metrics/particle/momentum/all/event/px_bias": 1.017188549041748, "metrics/particle/momentum/all/event/py_bias": 2.0128960609436035, "metrics/particle/momentum/all/event/pz_abs_mean": 13.576131820678711, "metrics/particle/momentum/all/event/pz_bias": 0.7737835645675659, "metrics/particle/momentum/all/event/pz_residual_rms": 25.20244026184082, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.01129353977739811, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04013494774699211, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.0247348602861166, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.00957084633409977, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06375159323215485, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0013054019073024392, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13706783950328827, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014154518023133278, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8454846739768982, "metrics/particle/momentum/class_1/accepted/samples": 16087.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.07362399995326996, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.16848649084568024, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.18508592247962952, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.1515732854604721, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.030351532623171806, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 4.533878803253174, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.005690912250429392, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.3298022747039795, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.002579587046056986, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1537857055664062, "metrics/particle/momentum/class_1/assigned/samples": 16869.0, "metrics/particle/momentum/class_1/efficiency": 0.9534167051315308, "metrics/particle/momentum/class_1/event/energy_bias": 3.645402669906616, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.672301292419434, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.124314785003662, "metrics/particle/momentum/class_1/event/px_bias": -0.7778722047805786, "metrics/particle/momentum/class_1/event/py_bias": -0.4658817648887634, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.658958435058594, "metrics/particle/momentum/class_1/event/pz_bias": -0.3897473216056824, "metrics/particle/momentum/class_1/event/pz_residual_rms": 20.966590881347656, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.043701205402612686, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20726235210895538, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.2098686546087265, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05833369120955467, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24964742362499237, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.031963471323251724, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7445075511932373, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.08436509221792221, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.8236091136932373, "metrics/particle/momentum/class_2/accepted/samples": 1095.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0180493593215942, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8442007899284363, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 13544.7783203125, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 13544.2099609375, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40909090638160706, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 687132.875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08741258829832077, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.5481114387512207, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.16604240238666534, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.096943855285645, "metrics/particle/momentum/class_2/assigned/samples": 2574.0, "metrics/particle/momentum/class_2/efficiency": 0.37169042229652405, "metrics/particle/momentum/class_2/event/energy_bias": -11.817513465881348, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.88910675048828, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.54991626739502, "metrics/particle/momentum/class_2/event/px_bias": 0.7485623955726624, "metrics/particle/momentum/class_2/event/py_bias": 1.009425163269043, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.600460052490234, "metrics/particle/momentum/class_2/event/pz_bias": 0.5900710225105286, "metrics/particle/momentum/class_2/event/pz_residual_rms": 19.816267013549805, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.030149510130286217, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14030422270298004, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.1407780796289444, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.03800853714346886, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18330837786197662, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0008019960951060057, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21520143747329712, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.014774193055927753, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7182953357696533, "metrics/particle/momentum/class_3/accepted/samples": 11222.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36157047748565674, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.6402009725570679, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.7363967895507812, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.6033014059066772, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18435564637184143, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 19.959266662597656, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.008645925670862198, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6204404234886169, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.019164567813277245, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 3.6783082485198975, "metrics/particle/momentum/class_3/assigned/samples": 14689.0, "metrics/particle/momentum/class_3/efficiency": 0.7281339168548584, "metrics/particle/momentum/class_3/event/energy_bias": 6.635115623474121, "metrics/particle/momentum/class_3/event/p3_error_mean": 11.13943099975586, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.931605815887451, "metrics/particle/momentum/class_3/event/px_bias": 0.7843578457832336, "metrics/particle/momentum/class_3/event/py_bias": 1.1339715719223022, "metrics/particle/momentum/class_3/event/pz_abs_mean": 8.255123138427734, "metrics/particle/momentum/class_3/event/pz_bias": 0.5127663612365723, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.998689651489258, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.015520079992711544, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.06184143200516701, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.06073978543281555, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.0179912056773901, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10757981240749359, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.35036414861679077, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.018871059641242027, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 1.073414921760559, "metrics/particle/momentum/class_4/accepted/samples": 336.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10117863863706589, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.2802720069885254, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.170214891433716, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 2.11205792427063, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 28.786104202270508, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4504178762435913, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.028973288834095, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.3148932456970215, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9281768202781677, "metrics/particle/momentum/class_4/event/energy_bias": -0.9331395626068115, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.126979351043701, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3829855918884277, "metrics/particle/momentum/class_4/event/px_bias": 0.11837217956781387, "metrics/particle/momentum/class_4/event/py_bias": 0.21572916209697723, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.349942684173584, "metrics/particle/momentum/class_4/event/pz_bias": 0.36194372177124023, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.4149274826049805, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.009944168850779533, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02417694963514805, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.021856889128684998, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006538202986121178, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06966560333967209, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.005649717524647713, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.27788859605789185, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.16323930025100708, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9518260359764099, "metrics/particle/momentum/class_5/accepted/samples": 177.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.04613557830452919, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.047278210520744324, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.0448024608194828, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0030874432995915413, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.02747252769768238, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.16523823142051697, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.016483517363667488, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.6684499979019165, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.3656674921512604, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.313662528991699, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9725274443626404, "metrics/particle/momentum/class_5/event/energy_bias": -0.7491343021392822, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.3493032455444336, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.9308171272277832, "metrics/particle/momentum/class_5/event/px_bias": 0.14376842975616455, "metrics/particle/momentum/class_5/event/py_bias": 0.1196519210934639, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.8038652539253235, "metrics/particle/momentum/class_5/event/pz_bias": -0.30125024914741516, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7702651023864746, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4453840255737305, "step_valid_time": 2603.963998556137, "step_total_time": 2605.409383058548, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029914322683444, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08274024364995447, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3028, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2855, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2841, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9469320066334992, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9428665785997358, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.94489491974185, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0571334214002642, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9422885572139303, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9382430647291942, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9402614595399635, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9469320066334992, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002763573190193, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09235653140206246, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2092, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2084, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9285397248113626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9372759856630825, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9328874024526198, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06272401433691756, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9249889036839769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9336917562724014, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9293199554069119, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9285397248113626, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0045007278113645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09436295441918208, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1272, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1149, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8996108949416343, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9088050314465409, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9041845913179507, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09119496855345911, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8941634241245137, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9033018867924528, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8987094251075479, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8996108949416343} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_7000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_7000.json new file mode 100644 index 0000000000000000000000000000000000000000..c23c1096df77d4dd56129b4aa17d23736e8cf87f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_7000.json @@ -0,0 +1 @@ +{"epoch": 0.3316278188364601, "train": {"Classification_binary": 2.9182469844818115, "Classification": 0.023628920316696167, "Regression_pt": 0.1519317626953125, "Regression_eta": 0.011030356399714947, "Regression_sin_phi": 0.009777205996215343, "Regression_cos_phi": 0.01028008759021759, "Regression_energy": 0.1545397937297821, "Total": 10.116904258728027, "Total_unweighted": 3.27943511120975}, "train_group": {}, "valid": {"Classification_binary": 2.889845530192057, "Classification": 0.02360346664985021, "Regression_pt": 0.15031160910924277, "Regression_eta": 0.011664409190416336, "Regression_sin_phi": 0.00906954767803351, "Regression_cos_phi": 0.009067777544260025, "Regression_energy": 0.15419148405392966, "Total": 9.968118031819662, "Total_unweighted": 3.2477538244177895, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02398645505309105, "diagnostic/loss/clic_pf/Classification_binary": 2.8869333267211914, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.00868058018386364, "diagnostic/loss/clic_pf/Regression_energy": 0.15665006637573242, "diagnostic/loss/clic_pf/Regression_eta": 0.010981966741383076, "diagnostic/loss/clic_pf/Regression_pt": 0.15313029289245605, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008803929202258587, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03179968148469925, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20696128904819489, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04735330492258072, "diagnostic/regression/clic_pf/energy_residual_rms": 0.46580132842063904, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.009216640144586563, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.20038117468357086, "diagnostic/regression/clic_pf/pt_residual_mean": 0.07806117087602615, "diagnostic/regression/clic_pf/pt_residual_rms": 0.47173652052879333, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9612380266189575, "metrics/particle/class_1/pid_accuracy": 0.9874061942100525, "metrics/particle/class_2/efficiency": 0.38294991850852966, "metrics/particle/class_2/pid_accuracy": 0.6755005717277527, "metrics/particle/class_3/efficiency": 0.7149046063423157, "metrics/particle/class_3/pid_accuracy": 0.9832670092582703, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.3169456124305725, "metrics/particle/class_5/efficiency": 0.9939637780189514, "metrics/particle/class_5/pid_accuracy": 0.5850202441215515, "metrics/particle/count/bias_mean": -1.4535948038101196, "metrics/particle/count/mae": 3.701960802078247, "metrics/particle/count/prediction_mean": 68.5457534790039, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.06251753866672516, "metrics/particle/event/energy_response_mean": 0.995976448059082, "metrics/particle/event/met_abs_error": 6.239057540893555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04876290261745453, "metrics/particle/event/scalar_pt_response_mean": 1.005733847618103, "metrics/particle/event/vector_pt_closure": 0.05044972896575928, "metrics/particle/matched/delta_eta_abs_mean": 0.014580778777599335, "metrics/particle/matched/delta_phi_abs_mean": 0.012768257409334183, "metrics/particle/matched/delta_r_mean": 0.021686259657144547, "metrics/particle/matched/energy_relative_abs_mean": 0.08716577291488647, "metrics/particle/matched/pid_accuracy": 0.9638304710388184, "metrics/particle/matched/pt_relative_abs_mean": 0.07489022612571716, "metrics/particle/matching/duplicate_fraction": 0.03184743598103523, "metrics/particle/matching/efficiency": 0.8064594268798828, "metrics/particle/matching/f1": 0.8149207234382629, "metrics/particle/matching/purity": 0.8235613703727722, "metrics/particle/matching/target_coverage_dr0p05": 0.7561601996421814, "metrics/particle/matching/target_coverage_dr0p10": 0.8390741348266602, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7207443714141846, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8480377793312073, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02470863237977028, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08268869668245316, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07247843593358994, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.008733497932553291, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12811164557933807, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0037268574815243483, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.349245548248291, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": 0.02743704803287983, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2527320384979248, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8318.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3290077745914459, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.1009564399719238, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.2929059267044067, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.1701722145080566, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13631068170070648, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 21.459558486938477, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.017864078283309937, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8641838431358337, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.10579478740692139, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 3.1084742546081543, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10300.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7738394141197205, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -2.8067705631256104, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 7.719940185546875, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.5244319438934326, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": -0.24993640184402466, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": -0.38213974237442017, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 6.586689472198486, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": 1.4804178476333618, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 10.741659164428711, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03454165160655975, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.0895128920674324, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.08357502520084381, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.012292365543544292, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14381767809391022, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01666666753590107, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.8895497918128967, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.050525739789009094, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6731607913970947, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1620.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.8352699279785156, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 2.6249985694885254, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 5.267292499542236, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 5.101003646850586, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.27120494842529297, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 142.3987274169922, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08336366713047028, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.3383946418762207, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13939280807971954, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.596960067749023, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2747.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.52734375, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.405897617340088, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.7245306968688965, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.1810826063156128, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.007985062897205353, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.025838010013103485, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.5760722160339355, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.09417149424552917, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 17.387920379638672, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.01937580667436123, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0887698158621788, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.0752466544508934, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.010509150102734566, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13160377740859985, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.00047588831512257457, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.12338778376579285, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": 0.03732398524880409, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5597789287567139, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18912.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.15964090824127197, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7504004836082458, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5411224365234375, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4358445107936859, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.08934900164604187, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.091734409332275, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0031710502225905657, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2775387167930603, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": 0.050801631063222885, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 2.3136982917785645, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21444.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8614375591278076, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.5873028039932251, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.340059280395508, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.546196460723877, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.31497037410736084, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -1.1771931648254395, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.567421913146973, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": 1.8460019826889038, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 14.364047050476074, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021764962002635002, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08705823868513107, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07491617649793625, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.010097328573465347, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13132765889167786, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.002322356915101409, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.23152877390384674, "metrics/particle/momentum/all/accepted/pz_bias": 0.0295404102653265, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.02914297580719, "metrics/particle/momentum/all/accepted/samples": 28850.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.26402848958969116, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 1.0043870210647583, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1.1420375108718872, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1.026687502861023, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11785683035850525, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 42.13747024536133, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013945667073130608, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6965065598487854, "metrics/particle/momentum/all/assigned/pz_bias": 0.0520763173699379, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.114005088806152, "metrics/particle/momentum/all/assigned/samples": 34491.0, "metrics/particle/momentum/all/efficiency": 0.8064290881156921, "metrics/particle/momentum/all/event/energy_bias": -5.625365734100342, "metrics/particle/momentum/all/event/p3_error_mean": 16.479446411132812, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.97149658203125, "metrics/particle/momentum/all/event/px_bias": -0.572891891002655, "metrics/particle/momentum/all/event/py_bias": -1.5334949493408203, "metrics/particle/momentum/all/event/pz_abs_mean": 12.698505401611328, "metrics/particle/momentum/all/event/pz_bias": 3.232248306274414, "metrics/particle/momentum/all/event/pz_residual_rms": 23.07567024230957, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.013205687515437603, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.04629009962081909, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.02367326058447361, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": -0.0012111898977309465, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06349145621061325, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0009877763222903013, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.13569888472557068, "metrics/particle/momentum/class_1/accepted/pz_bias": 0.03304417431354523, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7072509527206421, "metrics/particle/momentum/class_1/accepted/samples": 16198.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.0576871894299984, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.13030149042606354, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.1257641315460205, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.08718283474445343, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.024481328204274178, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 2.697795867919922, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004742146003991365, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.29859188199043274, "metrics/particle/momentum/class_1/assigned/pz_bias": 0.03225526586174965, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 3.1915485858917236, "metrics/particle/momentum/class_1/assigned/samples": 16870.0, "metrics/particle/momentum/class_1/efficiency": 0.9599952697753906, "metrics/particle/momentum/class_1/event/energy_bias": 1.9097952842712402, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.516733169555664, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.811256408691406, "metrics/particle/momentum/class_1/event/px_bias": -0.9863409996032715, "metrics/particle/momentum/class_1/event/py_bias": -0.5028830170631409, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.829522132873535, "metrics/particle/momentum/class_1/event/pz_bias": 1.3174347877502441, "metrics/particle/momentum/class_1/event/pz_residual_rms": 21.036954879760742, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04379083588719368, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20328041911125183, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20418033003807068, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05064031854271889, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.2430233657360077, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.035650622099637985, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7308217287063599, "metrics/particle/momentum/class_2/accepted/pz_bias": 0.10042398422956467, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.7784690856933594, "metrics/particle/momentum/class_2/accepted/samples": 1122.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0210040807724, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8352555632591248, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.9006778597831726, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.37313517928123474, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.3844936788082123, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 3.2890868186950684, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08662974834442139, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4751193523406982, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.1886587142944336, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.161920547485352, "metrics/particle/momentum/class_2/assigned/samples": 2528.0, "metrics/particle/momentum/class_2/efficiency": 0.38085541129112244, "metrics/particle/momentum/class_2/event/energy_bias": -10.509017944335938, "metrics/particle/momentum/class_2/event/p3_error_mean": 16.422025680541992, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 8.206356048583984, "metrics/particle/momentum/class_2/event/px_bias": -0.09475068747997284, "metrics/particle/momentum/class_2/event/py_bias": -1.599331021308899, "metrics/particle/momentum/class_2/event/pz_abs_mean": 12.442254066467285, "metrics/particle/momentum/class_2/event/pz_bias": 1.6256901025772095, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.84443473815918, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03242386132478714, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.1371917575597763, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13857802748680115, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.033201202750205994, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.18132486939430237, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0009985475335270166, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21798200905323029, "metrics/particle/momentum/class_3/accepted/pz_bias": 0.01576327160000801, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7451332807540894, "metrics/particle/momentum/class_3/accepted/samples": 11016.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.3785261809825897, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 2.0638515949249268, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.39361572265625, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 2.2584805488586426, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1826929748058319, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 64.79764556884766, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.012234517373144627, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.682107150554657, "metrics/particle/momentum/class_3/assigned/pz_bias": 0.05467585101723671, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.052994728088379, "metrics/particle/momentum/class_3/assigned/samples": 14549.0, "metrics/particle/momentum/class_3/efficiency": 0.7147676944732666, "metrics/particle/momentum/class_3/event/energy_bias": 4.884214878082275, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.248312950134277, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.409623146057129, "metrics/particle/momentum/class_3/event/px_bias": 0.3003848195075989, "metrics/particle/momentum/class_3/event/py_bias": 0.5299320220947266, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.595760345458984, "metrics/particle/momentum/class_3/event/pz_bias": 0.07930091768503189, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.3701171875, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.01671341247856617, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.0517343245446682, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05277848616242409, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.007935239002108574, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.0953965038061142, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.24312277138233185, "metrics/particle/momentum/class_4/accepted/pz_bias": 0.08404522389173508, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.7470290064811707, "metrics/particle/momentum/class_4/accepted/samples": 333.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.10597368329763412, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.8271439671516418, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.4451450705528259, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.3878552317619324, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05801105126738548, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.124279022216797, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.008287292905151844, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.4544833302497864, "metrics/particle/momentum/class_4/assigned/pz_bias": 0.08830414712429047, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.9392337799072266, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.919889509677887, "metrics/particle/momentum/class_4/event/energy_bias": -0.9169645309448242, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.0746402740478516, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3475072383880615, "metrics/particle/momentum/class_4/event/px_bias": 0.0971069261431694, "metrics/particle/momentum/class_4/event/py_bias": -0.007127687335014343, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3333549499511719, "metrics/particle/momentum/class_4/event/pz_bias": 0.4242187440395355, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.741910934448242, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.011785782873630524, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.028788268566131592, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.025586724281311035, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": -0.0035451429430395365, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.06605727225542068, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.3166992962360382, "metrics/particle/momentum/class_5/accepted/pz_bias": 0.01480873953551054, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8571121692657471, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.03728121146559715, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03405805304646492, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.030795689672231674, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": -0.008875216357409954, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.09771391749382019, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.617358386516571, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.28767186403274536, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.168170928955078, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.993392288684845, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.2590875625610352, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.838560938835144, "metrics/particle/momentum/class_5/event/px_bias": 0.11070813238620758, "metrics/particle/momentum/class_5/event/py_bias": 0.045914649963378906, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7840602397918701, "metrics/particle/momentum/class_5/event/pz_bias": -0.21439620852470398, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6929829120635986, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.4436092376708984, "step_valid_time": 3035.524728536606, "step_total_time": 3036.9683384895325, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9942889103976454, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07601531204541334, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3002, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2837, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2827, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9409618573797678, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9450366422385077, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9429948479308625, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05496335776149234, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9376451077943615, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.941705529646902, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9396709323583181, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9409618573797678, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9954659679561199, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09118473963230034, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2232, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2079, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2071, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9227696404793608, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9314516129032258, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9270903010033444, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06854838709677419, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9192188193519751, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9278673835125448, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9235228539576366, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9227696404793608, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9979281651145155, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08081777387756306, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1274, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1161, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1156, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9035019455252918, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9113029827315542, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9073856975381008, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08869701726844584, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8996108949416343, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9073783359497645, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9034779210629152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9035019455252918} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_8000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_8000.json new file mode 100644 index 0000000000000000000000000000000000000000..b14f7d0b25d668552d410679de3fea6be4ec255b --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_8000.json @@ -0,0 +1 @@ +{"epoch": 0.37900322152738297, "train": {"Classification_binary": 2.8568334579467773, "Classification": 0.02579425275325775, "Regression_pt": 0.14918678998947144, "Regression_eta": 0.009606762789189816, "Regression_sin_phi": 0.008224865421652794, "Regression_cos_phi": 0.007899274118244648, "Regression_energy": 0.1533612608909607, "Total": 9.771207809448242, "Total_unweighted": 3.2109066639095545}, "train_group": {}, "valid": {"Classification_binary": 2.8574097951253257, "Classification": 0.0236475666364034, "Regression_pt": 0.1500506599744161, "Regression_eta": 0.011586775382359823, "Regression_sin_phi": 0.009096071744958559, "Regression_cos_phi": 0.009038938830296198, "Regression_energy": 0.15373300512631735, "Total": 9.923037846883139, "Total_unweighted": 3.214562812820077, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.024192923679947853, "diagnostic/loss/clic_pf/Classification_binary": 2.851804256439209, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008688561618328094, "diagnostic/loss/clic_pf/Regression_energy": 0.15596558153629303, "diagnostic/loss/clic_pf/Regression_eta": 0.01080928836017847, "diagnostic/loss/clic_pf/Regression_pt": 0.15302133560180664, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.008836330845952034, "diagnostic/regression/clic_pf/energy_pred_mean": 0.03327179700136185, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.2015499472618103, "diagnostic/regression/clic_pf/energy_residual_mean": 0.04882542043924332, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4599980413913727, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": -0.0033178830053657293, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.1946490854024887, "diagnostic/regression/clic_pf/pt_residual_mean": 0.06552664935588837, "diagnostic/regression/clic_pf/pt_residual_rms": 0.46232175827026367, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.9633252024650574, "metrics/particle/class_1/pid_accuracy": 0.9870620369911194, "metrics/particle/class_2/efficiency": 0.39422643184661865, "metrics/particle/class_2/pid_accuracy": 0.717391312122345, "metrics/particle/class_3/efficiency": 0.7351223230361938, "metrics/particle/class_3/pid_accuracy": 0.9801501035690308, "metrics/particle/class_4/efficiency": 0.9263566136360168, "metrics/particle/class_4/pid_accuracy": 0.2447698712348938, "metrics/particle/class_5/efficiency": 0.9879275560379028, "metrics/particle/class_5/pid_accuracy": 0.6232179403305054, "metrics/particle/count/bias_mean": 0.2450980395078659, "metrics/particle/count/mae": 3.3967320919036865, "metrics/particle/count/prediction_mean": 70.24444580078125, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.057760290801525116, "metrics/particle/event/energy_response_mean": 1.0141139030456543, "metrics/particle/event/met_abs_error": 6.403356552124023, "metrics/particle/event/scalar_pt_relative_abs_error": 0.046336714178323746, "metrics/particle/event/scalar_pt_response_mean": 1.0124881267547607, "metrics/particle/event/vector_pt_closure": 0.049801215529441833, "metrics/particle/matched/delta_eta_abs_mean": 0.013516931794583797, "metrics/particle/matched/delta_phi_abs_mean": 0.01317768357694149, "metrics/particle/matched/delta_r_mean": 0.021167483180761337, "metrics/particle/matched/energy_relative_abs_mean": 0.08606589585542679, "metrics/particle/matched/pid_accuracy": 0.9634445905685425, "metrics/particle/matched/pt_relative_abs_mean": 0.07440536469221115, "metrics/particle/matching/duplicate_fraction": 0.0329568088054657, "metrics/particle/matching/efficiency": 0.8171038031578064, "metrics/particle/matching/f1": 0.8156757950782776, "metrics/particle/matching/purity": 0.8142527341842651, "metrics/particle/matching/target_coverage_dr0p05": 0.7431722283363342, "metrics/particle/matching/target_coverage_dr0p10": 0.8480377793312073, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.732154369354248, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8610537648200989, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.02513427473604679, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.08216670155525208, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.07288375496864319, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.003946817014366388, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.12877891957759857, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.0033175356220453978, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3325682580471039, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.036263156682252884, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.255993366241455, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8440.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.3164273202419281, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.109688401222229, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.3470039367675781, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.214013934135437, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.13549435138702393, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 22.001502990722656, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015203671529889107, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.8508915901184082, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": -0.010675961151719093, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 5.630108833312988, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10458.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7851893305778503, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": 0.4711729884147644, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.24244213104248, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.657245635986328, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.3131241798400879, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.066465362906456, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.076836585998535, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.7100517749786377, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 14.388960838317871, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03194797411561012, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.09078699350357056, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0849166214466095, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.010924515314400196, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14243561029434204, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.016552511602640152, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.829055905342102, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.027228010818362236, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.690962314605713, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1752.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7996250987052917, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.4175162315368652, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 13916.935546875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 13916.748046875, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2737603187561035, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 749776.4375, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.0819559246301651, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.27235746383667, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.13626456260681152, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 10.55257511138916, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2904.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5703125, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -0.5594491958618164, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 8.226221084594727, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.325597882270813, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": -0.2555796205997467, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.0658002495765686, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 8.045205116271973, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -2.2849676609039307, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 19.16082000732422, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.018560949712991714, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.0878438651561737, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07497824728488922, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.008977255783975124, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13175508379936218, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0006832036888226867, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.11545925587415695, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.024051474407315254, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.616855800151825, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 19028.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1636064201593399, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7601593136787415, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5965577363967896, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4865327775478363, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09100993722677231, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.982237815856934, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0032354979775846004, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.26554667949676514, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.05233810096979141, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.346602201461792, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21635.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8667213320732117, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 1.4016395807266235, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 9.054089546203613, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 7.354001522064209, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.43384310603141785, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": 0.14112529158592224, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.718520164489746, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.3001783490180969, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 22.34684181213379, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021262282505631447, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.08638051897287369, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07496915757656097, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.007641003001481295, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13157136738300323, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0023956194054335356, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2209562063217163, "metrics/particle/momentum/all/accepted/pz_bias": -0.027769196778535843, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0665942430496216, "metrics/particle/momentum/all/accepted/samples": 29220.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2620491087436676, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9191540479660034, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 1155.5782470703125, "metrics/particle/momentum/all/assigned/pt_relative_bias": 1155.455078125, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11946738511323929, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 215980.578125, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.013344001024961472, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6899635195732117, "metrics/particle/momentum/all/assigned/pz_bias": -0.0468524806201458, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.453474521636963, "metrics/particle/momentum/all/assigned/samples": 34997.0, "metrics/particle/momentum/all/efficiency": 0.8167715072631836, "metrics/particle/momentum/all/event/energy_bias": 1.3133634328842163, "metrics/particle/momentum/all/event/p3_error_mean": 17.703784942626953, "metrics/particle/momentum/all/event/pt_vector_error_mean": 8.818612098693848, "metrics/particle/momentum/all/event/px_bias": -0.37629854679107666, "metrics/particle/momentum/all/event/py_bias": 0.27339088916778564, "metrics/particle/momentum/all/event/pz_abs_mean": 13.428882598876953, "metrics/particle/momentum/all/event/pz_bias": -4.2951979637146, "metrics/particle/momentum/all/event/pz_residual_rms": 28.707975387573242, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.00980325136333704, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.0414273627102375, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022848432883620262, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.001893399516120553, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.0638105645775795, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0011703110067173839, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11726564168930054, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.014437288977205753, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.7764001488685608, "metrics/particle/momentum/class_1/accepted/samples": 16235.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.052000001072883606, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.14609840512275696, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.15822821855545044, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.123805470764637, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.022521188482642174, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 6.119316577911377, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004444971214979887, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.26302507519721985, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.01750289462506771, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.3924560546875, "metrics/particle/momentum/class_1/assigned/samples": 16873.0, "metrics/particle/momentum/class_1/efficiency": 0.9621881246566772, "metrics/particle/momentum/class_1/event/energy_bias": 2.9963557720184326, "metrics/particle/momentum/class_1/event/p3_error_mean": 11.415496826171875, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 6.037086486816406, "metrics/particle/momentum/class_1/event/px_bias": -1.333208441734314, "metrics/particle/momentum/class_1/event/py_bias": -0.4275566339492798, "metrics/particle/momentum/class_1/event/pz_abs_mean": 8.55831241607666, "metrics/particle/momentum/class_1/event/pz_bias": -0.13307884335517883, "metrics/particle/momentum/class_1/event/pz_residual_rms": 25.38862419128418, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04161874204874039, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.20735085010528564, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.20853851735591888, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.05122983083128929, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.24812456965446472, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.032843559980392456, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.7199476957321167, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.1288134902715683, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.746145725250244, "metrics/particle/momentum/class_2/accepted/samples": 1157.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0189143419265747, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8071644306182861, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 14960.068359375, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 14959.5126953125, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.40096262097358704, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 777441.6875, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08626434952020645, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.5621745586395264, "metrics/particle/momentum/class_2/assigned/pz_bias": -0.2908991277217865, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 9.089211463928223, "metrics/particle/momentum/class_2/assigned/samples": 2701.0, "metrics/particle/momentum/class_2/efficiency": 0.39273589849472046, "metrics/particle/momentum/class_2/event/energy_bias": -5.880516052246094, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.552155494689941, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.808946132659912, "metrics/particle/momentum/class_2/event/px_bias": -0.3177180886268616, "metrics/particle/momentum/class_2/event/py_bias": 0.05473518371582031, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.65897274017334, "metrics/particle/momentum/class_2/event/pz_bias": -3.959801435470581, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.50154685974121, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03606614097952843, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14051663875579834, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.13760507106781006, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.02223174087703228, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.17943347990512848, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.001060726586729288, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.21538980305194855, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.035135433077812195, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7752332091331482, "metrics/particle/momentum/class_3/accepted/samples": 11313.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.36955174803733826, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8286988735198975, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 2.1343677043914795, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.9859436750411987, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.1811949759721756, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 28.331527709960938, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.010484575293958187, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.6631451845169067, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03132602572441101, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.992973327636719, "metrics/particle/momentum/class_3/assigned/samples": 14879.0, "metrics/particle/momentum/class_3/efficiency": 0.7340384125709534, "metrics/particle/momentum/class_3/event/energy_bias": 5.898252964019775, "metrics/particle/momentum/class_3/event/p3_error_mean": 10.200698852539062, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.452818870544434, "metrics/particle/momentum/class_3/event/px_bias": 1.0889365673065186, "metrics/particle/momentum/class_3/event/py_bias": 0.45073172450065613, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.447829246520996, "metrics/particle/momentum/class_3/event/pz_bias": -0.299436092376709, "metrics/particle/momentum/class_3/event/pz_residual_rms": 12.216395378112793, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.013609608635306358, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.050408367067575455, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05038859695196152, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": -0.0020705240312963724, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.09820087254047394, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.2521689832210541, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.02601148933172226, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9081324934959412, "metrics/particle/momentum/class_4/accepted/samples": 335.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.11113443970680237, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 0.8460190296173096, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 0.5430997014045715, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 0.48766592144966125, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.05524861812591553, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 3.5088815689086914, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0027624310459941626, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.3460102379322052, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04888660088181496, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 1.1435893774032593, "metrics/particle/momentum/class_4/assigned/samples": 362.0, "metrics/particle/momentum/class_4/efficiency": 0.9254143834114075, "metrics/particle/momentum/class_4/event/energy_bias": -1.118561029434204, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.1013550758361816, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.3581905364990234, "metrics/particle/momentum/class_4/event/px_bias": 0.09604018926620483, "metrics/particle/momentum/class_4/event/py_bias": 0.12601014971733093, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3306015729904175, "metrics/particle/momentum/class_4/event/pz_bias": 0.3325980305671692, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.739478588104248, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.007776263169944286, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02782762236893177, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.026495495811104774, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.005497953854501247, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.07538604736328125, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0055555556900799274, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.22984595596790314, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.12104639410972595, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.8907357454299927, "metrics/particle/momentum/class_5/accepted/samples": 180.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.014606517739593983, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.03788527846336365, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.036499492824077606, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0051178960129618645, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.12359213829040527, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.010989011265337467, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5219933986663818, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.41128918528556824, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 4.052502632141113, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9890109896659851, "metrics/particle/momentum/class_5/event/energy_bias": -0.5821682810783386, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.1217997074127197, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.742102324962616, "metrics/particle/momentum/class_5/event/px_bias": 0.08965110033750534, "metrics/particle/momentum/class_5/event/py_bias": 0.06947045028209686, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7107675075531006, "metrics/particle/momentum/class_5/event/pz_bias": -0.2354794442653656, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.6056056022644043, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5246233940124512, "step_valid_time": 3466.2710752487183, "step_total_time": 3467.7956988811493, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9969761169996396, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07464655017222876, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3019, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2862, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2850, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9492537313432836, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9479960251738987, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9486244613854823, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.052003974826101355, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.945273631840796, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9440211990725406, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9446470003314551, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9492537313432836, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005834266760112, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08483272285110433, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2236, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2091, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2084, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9280958721704394, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9351520572450805, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9316106036979283, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0648479427549195, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9249889036839769, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9320214669051878, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9284918690131433, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9280958721704394, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004639292922453, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08799890993218762, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1278, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1158, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1150, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9011673151750973, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9061032863849765, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9036285602809208, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09389671361502347, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8949416342412452, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8998435054773083, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8973858759266484, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9011673151750973} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_9000.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_9000.json new file mode 100644 index 0000000000000000000000000000000000000000..fc75c267dc91e22fda70a6fe99266672ce1dd4fc --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/history/step_9000.json @@ -0,0 +1 @@ +{"epoch": 0.42637862421830586, "train": {"Classification_binary": 2.737908124923706, "Classification": 0.022603344172239304, "Regression_pt": 0.1359359622001648, "Regression_eta": 0.009333481080830097, "Regression_sin_phi": 0.007729464676231146, "Regression_cos_phi": 0.008754482492804527, "Regression_energy": 0.13509508967399597, "Total": 9.08202075958252, "Total_unweighted": 3.057359949219972}, "train_group": {}, "valid": {"Classification_binary": 2.8073463439941406, "Classification": 0.022935273746649425, "Regression_pt": 0.14622191588083902, "Regression_eta": 0.011398473133643469, "Regression_sin_phi": 0.008966248482465744, "Regression_cos_phi": 0.008971353992819786, "Regression_energy": 0.1497175395488739, "Total": 9.708833694458008, "Total_unweighted": 3.155557148779432, "diagnostic/composition/clic_pf/events": 128.0, "diagnostic/composition/clic_pf/target_particles": 8943.75, "diagnostic/composition/clic_pf/valid_elements": 13648.5, "diagnostic/loss/clic_pf/Classification": 0.02347753569483757, "diagnostic/loss/clic_pf/Classification_binary": 2.808593273162842, "diagnostic/loss/clic_pf/Regression_cos_phi": 0.008467447012662888, "diagnostic/loss/clic_pf/Regression_energy": 0.15321855247020721, "diagnostic/loss/clic_pf/Regression_eta": 0.010784109123051167, "diagnostic/loss/clic_pf/Regression_pt": 0.14999240636825562, "diagnostic/loss/clic_pf/Regression_sin_phi": 0.00872043240815401, "diagnostic/regression/clic_pf/energy_pred_mean": 0.04032832011580467, "diagnostic/regression/clic_pf/energy_residual_abs_mean": 0.20056505501270294, "diagnostic/regression/clic_pf/energy_residual_mean": 0.05588194355368614, "diagnostic/regression/clic_pf/energy_residual_rms": 0.4515126943588257, "diagnostic/regression/clic_pf/energy_target_mean": -0.015553624369204044, "diagnostic/regression/clic_pf/pt_pred_mean": 0.013512368313968182, "diagnostic/regression/clic_pf/pt_residual_abs_mean": 0.19594918191432953, "diagnostic/regression/clic_pf/pt_residual_mean": 0.08235690742731094, "diagnostic/regression/clic_pf/pt_residual_rms": 0.45545223355293274, "diagnostic/regression/clic_pf/pt_target_mean": -0.06884454190731049, "metrics/particle/class_1/efficiency": 0.963245689868927, "metrics/particle/class_1/pid_accuracy": 0.9861323237419128, "metrics/particle/class_2/efficiency": 0.38644564151763916, "metrics/particle/class_2/pid_accuracy": 0.705865204334259, "metrics/particle/class_3/efficiency": 0.7135035991668701, "metrics/particle/class_3/pid_accuracy": 0.9817237257957458, "metrics/particle/class_4/efficiency": 0.9244186282157898, "metrics/particle/class_4/pid_accuracy": 0.3616352081298828, "metrics/particle/class_5/efficiency": 0.9919517040252686, "metrics/particle/class_5/pid_accuracy": 0.6673427820205688, "metrics/particle/count/bias_mean": -1.6620914936065674, "metrics/particle/count/mae": 3.688235282897949, "metrics/particle/count/prediction_mean": 68.3372573852539, "metrics/particle/count/target_mean": 69.99934387207031, "metrics/particle/event/energy_relative_abs_error": 0.05369831994175911, "metrics/particle/event/energy_response_mean": 0.9981302618980408, "metrics/particle/event/met_abs_error": 5.574217319488525, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04348813369870186, "metrics/particle/event/scalar_pt_response_mean": 1.0073521137237549, "metrics/particle/event/vector_pt_closure": 0.04711511731147766, "metrics/particle/matched/delta_eta_abs_mean": 0.0149993309751153, "metrics/particle/matched/delta_phi_abs_mean": 0.012721700593829155, "metrics/particle/matched/delta_r_mean": 0.02180829457938671, "metrics/particle/matched/energy_relative_abs_mean": 0.08520098775625229, "metrics/particle/matched/pid_accuracy": 0.96462082862854, "metrics/particle/matched/pt_relative_abs_mean": 0.07623854279518127, "metrics/particle/matching/duplicate_fraction": 0.03064386546611786, "metrics/particle/matching/efficiency": 0.8070570230484009, "metrics/particle/matching/f1": 0.8167536854743958, "metrics/particle/matching/purity": 0.8266862034797668, "metrics/particle/matching/target_coverage_dr0p05": 0.7405951619148254, "metrics/particle/matching/target_coverage_dr0p10": 0.8396623730659485, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7135080695152283, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8477389812469482, "metrics/particle/momentum/abs_eta_1to2/accepted/delta_r_mean": 0.023721957579255104, "metrics/particle/momentum/abs_eta_1to2/accepted/energy_relative_abs_mean": 0.07912981510162354, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_abs_mean": 0.0752997174859047, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_bias": 0.025392092764377594, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_1to2/accepted/pt_relative_residual_rms": 0.1326674371957779, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_gt10_fraction": 0.003730894299224019, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_abs_mean": 0.3206053376197815, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_bias": -0.008272867649793625, "metrics/particle/momentum/abs_eta_1to2/accepted/pz_residual_rms": 1.2233624458312988, "metrics/particle/momentum/abs_eta_1to2/accepted/samples": 8309.0, "metrics/particle/momentum/abs_eta_1to2/assigned/delta_r_mean": 0.31469932198524475, "metrics/particle/momentum/abs_eta_1to2/assigned/energy_relative_abs_mean": 1.0760539770126343, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_abs_mean": 1.1525746583938599, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_bias": 1.0407460927963257, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_gt0p5_fraction": 0.1350165754556656, "metrics/particle/momentum/abs_eta_1to2/assigned/pt_relative_residual_rms": 10.86752700805664, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_gt10_fraction": 0.015110158361494541, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_abs_mean": 0.836847722530365, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_bias": 0.03362789377570152, "metrics/particle/momentum/abs_eta_1to2/assigned/pz_residual_rms": 4.302870750427246, "metrics/particle/momentum/abs_eta_1to2/assigned/samples": 10258.0, "metrics/particle/momentum/abs_eta_1to2/efficiency": 0.7730021476745605, "metrics/particle/momentum/abs_eta_1to2/event/energy_bias": -0.23837172985076904, "metrics/particle/momentum/abs_eta_1to2/event/p3_error_mean": 8.828144073486328, "metrics/particle/momentum/abs_eta_1to2/event/pt_vector_error_mean": 3.967787742614746, "metrics/particle/momentum/abs_eta_1to2/event/px_bias": 0.31490328907966614, "metrics/particle/momentum/abs_eta_1to2/event/py_bias": 0.46065375208854675, "metrics/particle/momentum/abs_eta_1to2/event/pz_abs_mean": 7.633419036865234, "metrics/particle/momentum/abs_eta_1to2/event/pz_bias": -1.1078472137451172, "metrics/particle/momentum/abs_eta_1to2/event/pz_residual_rms": 16.208742141723633, "metrics/particle/momentum/abs_eta_1to2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_1to2/target_samples": 10749.0, "metrics/particle/momentum/abs_eta_ge2/accepted/delta_r_mean": 0.03278150409460068, "metrics/particle/momentum/abs_eta_ge2/accepted/energy_relative_abs_mean": 0.08437702059745789, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_abs_mean": 0.0868878960609436, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_bias": 0.030675480142235756, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_ge2/accepted/pt_relative_residual_rms": 0.14501377940177917, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_gt10_fraction": 0.01379724033176899, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_abs_mean": 0.7717686891555786, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_bias": -0.06293267756700516, "metrics/particle/momentum/abs_eta_ge2/accepted/pz_residual_rms": 2.6297876834869385, "metrics/particle/momentum/abs_eta_ge2/accepted/samples": 1667.0, "metrics/particle/momentum/abs_eta_ge2/assigned/delta_r_mean": 0.7919918298721313, "metrics/particle/momentum/abs_eta_ge2/assigned/energy_relative_abs_mean": 1.9729981422424316, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_abs_mean": 3.184481382369995, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_bias": 3.0238897800445557, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_gt0p5_fraction": 0.2681950628757477, "metrics/particle/momentum/abs_eta_ge2/assigned/pt_relative_residual_rms": 31.176042556762695, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_gt10_fraction": 0.08042212575674057, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_abs_mean": 3.2080376148223877, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_bias": -0.04760650172829628, "metrics/particle/momentum/abs_eta_ge2/assigned/pz_residual_rms": 11.159984588623047, "metrics/particle/momentum/abs_eta_ge2/assigned/samples": 2748.0, "metrics/particle/momentum/abs_eta_ge2/efficiency": 0.5426432490348816, "metrics/particle/momentum/abs_eta_ge2/event/energy_bias": -3.3923048973083496, "metrics/particle/momentum/abs_eta_ge2/event/p3_error_mean": 7.375409126281738, "metrics/particle/momentum/abs_eta_ge2/event/pt_vector_error_mean": 1.2668272256851196, "metrics/particle/momentum/abs_eta_ge2/event/px_bias": 0.04898390918970108, "metrics/particle/momentum/abs_eta_ge2/event/py_bias": 0.18921342492103577, "metrics/particle/momentum/abs_eta_ge2/event/pz_abs_mean": 7.195428371429443, "metrics/particle/momentum/abs_eta_ge2/event/pz_bias": -0.7754809856414795, "metrics/particle/momentum/abs_eta_ge2/event/pz_residual_rms": 16.886079788208008, "metrics/particle/momentum/abs_eta_ge2/event/samples": 512.0, "metrics/particle/momentum/abs_eta_ge2/target_samples": 3072.0, "metrics/particle/momentum/abs_eta_lt1/accepted/delta_r_mean": 0.0200558640062809, "metrics/particle/momentum/abs_eta_lt1/accepted/energy_relative_abs_mean": 0.08695152401924133, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_abs_mean": 0.07507528364658356, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_bias": 0.021104758605360985, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/abs_eta_lt1/accepted/pt_relative_residual_rms": 0.13280630111694336, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_gt10_fraction": 0.0005311802960932255, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_abs_mean": 0.1186886578798294, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_bias": -0.03857263922691345, "metrics/particle/momentum/abs_eta_lt1/accepted/pz_residual_rms": 0.5842148661613464, "metrics/particle/momentum/abs_eta_lt1/accepted/samples": 18826.0, "metrics/particle/momentum/abs_eta_lt1/assigned/delta_r_mean": 0.1624150425195694, "metrics/particle/momentum/abs_eta_lt1/assigned/energy_relative_abs_mean": 0.7376573085784912, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_abs_mean": 0.5521350502967834, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_bias": 0.4548567533493042, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_gt0p5_fraction": 0.09275510907173157, "metrics/particle/momentum/abs_eta_lt1/assigned/pt_relative_residual_rms": 6.477052211761475, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_gt10_fraction": 0.0029875829350203276, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_abs_mean": 0.2712038457393646, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_bias": -0.026563839986920357, "metrics/particle/momentum/abs_eta_lt1/assigned/pz_residual_rms": 1.5193425416946411, "metrics/particle/momentum/abs_eta_lt1/assigned/samples": 21422.0, "metrics/particle/momentum/abs_eta_lt1/efficiency": 0.8575202822685242, "metrics/particle/momentum/abs_eta_lt1/event/energy_bias": 0.26585516333580017, "metrics/particle/momentum/abs_eta_lt1/event/p3_error_mean": 8.201701164245605, "metrics/particle/momentum/abs_eta_lt1/event/pt_vector_error_mean": 6.607175827026367, "metrics/particle/momentum/abs_eta_lt1/event/px_bias": -0.15717366337776184, "metrics/particle/momentum/abs_eta_lt1/event/py_bias": -0.9207809567451477, "metrics/particle/momentum/abs_eta_lt1/event/pz_abs_mean": 4.253774642944336, "metrics/particle/momentum/abs_eta_lt1/event/pz_bias": -0.038674548268318176, "metrics/particle/momentum/abs_eta_lt1/event/pz_residual_rms": 10.415706634521484, "metrics/particle/momentum/abs_eta_lt1/event/samples": 512.0, "metrics/particle/momentum/abs_eta_lt1/target_samples": 21954.0, "metrics/particle/momentum/all/accepted/delta_r_mean": 0.021850017830729485, "metrics/particle/momentum/all/accepted/energy_relative_abs_mean": 0.0845460444688797, "metrics/particle/momentum/all/accepted/pt_relative_abs_mean": 0.07582371681928635, "metrics/particle/momentum/all/accepted/pt_relative_bias": 0.022895529866218567, "metrics/particle/momentum/all/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/all/accepted/pt_relative_residual_rms": 0.13350345194339752, "metrics/particle/momentum/all/accepted/pz_abs_gt10_fraction": 0.0022220679093152285, "metrics/particle/momentum/all/accepted/pz_abs_mean": 0.2147378921508789, "metrics/particle/momentum/all/accepted/pz_bias": -0.031241459771990776, "metrics/particle/momentum/all/accepted/pz_residual_rms": 1.0271881818771362, "metrics/particle/momentum/all/accepted/samples": 28802.0, "metrics/particle/momentum/all/assigned/delta_r_mean": 0.2580409646034241, "metrics/particle/momentum/all/assigned/energy_relative_abs_mean": 0.9370876550674438, "metrics/particle/momentum/all/assigned/pt_relative_abs_mean": 0.9411497712135315, "metrics/particle/momentum/all/assigned/pt_relative_bias": 0.8344824910163879, "metrics/particle/momentum/all/assigned/pt_relative_gt0p5_fraction": 0.11935053020715714, "metrics/particle/momentum/all/assigned/pt_relative_residual_rms": 11.784425735473633, "metrics/particle/momentum/all/assigned/pz_abs_gt10_fraction": 0.012780294753611088, "metrics/particle/momentum/all/assigned/pz_abs_mean": 0.6741546392440796, "metrics/particle/momentum/all/assigned/pz_bias": -0.010309001430869102, "metrics/particle/momentum/all/assigned/pz_residual_rms": 4.11022424697876, "metrics/particle/momentum/all/assigned/samples": 34428.0, "metrics/particle/momentum/all/efficiency": 0.8050873279571533, "metrics/particle/momentum/all/event/energy_bias": -3.364821434020996, "metrics/particle/momentum/all/event/p3_error_mean": 15.452238082885742, "metrics/particle/momentum/all/event/pt_vector_error_mean": 7.8245134353637695, "metrics/particle/momentum/all/event/px_bias": 0.2067135125398636, "metrics/particle/momentum/all/event/py_bias": -0.2709137499332428, "metrics/particle/momentum/all/event/pz_abs_mean": 11.607745170593262, "metrics/particle/momentum/all/event/pz_bias": -1.9220027923583984, "metrics/particle/momentum/all/event/pz_residual_rms": 18.01988983154297, "metrics/particle/momentum/all/event/samples": 512.0, "metrics/particle/momentum/all/target_samples": 35775.0, "metrics/particle/momentum/class_1/accepted/delta_r_mean": 0.011317315511405468, "metrics/particle/momentum/class_1/accepted/energy_relative_abs_mean": 0.03955022618174553, "metrics/particle/momentum/class_1/accepted/pt_relative_abs_mean": 0.022003542631864548, "metrics/particle/momentum/class_1/accepted/pt_relative_bias": 0.004866966977715492, "metrics/particle/momentum/class_1/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_1/accepted/pt_relative_residual_rms": 0.06318862736225128, "metrics/particle/momentum/class_1/accepted/pz_abs_gt10_fraction": 0.0010473139118403196, "metrics/particle/momentum/class_1/accepted/pz_abs_mean": 0.11910510063171387, "metrics/particle/momentum/class_1/accepted/pz_bias": -0.023484326899051666, "metrics/particle/momentum/class_1/accepted/pz_residual_rms": 0.8211575150489807, "metrics/particle/momentum/class_1/accepted/samples": 16232.0, "metrics/particle/momentum/class_1/assigned/delta_r_mean": 0.05693880096077919, "metrics/particle/momentum/class_1/assigned/energy_relative_abs_mean": 0.12379530817270279, "metrics/particle/momentum/class_1/assigned/pt_relative_abs_mean": 0.10659120976924896, "metrics/particle/momentum/class_1/assigned/pt_relative_bias": 0.07632457464933395, "metrics/particle/momentum/class_1/assigned/pt_relative_gt0p5_fraction": 0.023950675502419472, "metrics/particle/momentum/class_1/assigned/pt_relative_residual_rms": 1.5988433361053467, "metrics/particle/momentum/class_1/assigned/pz_abs_gt10_fraction": 0.004624140448868275, "metrics/particle/momentum/class_1/assigned/pz_abs_mean": 0.2651768624782562, "metrics/particle/momentum/class_1/assigned/pz_bias": -0.023005517199635506, "metrics/particle/momentum/class_1/assigned/pz_residual_rms": 2.414968490600586, "metrics/particle/momentum/class_1/assigned/samples": 16868.0, "metrics/particle/momentum/class_1/efficiency": 0.9620103240013123, "metrics/particle/momentum/class_1/event/energy_bias": 0.5881491899490356, "metrics/particle/momentum/class_1/event/p3_error_mean": 10.31421184539795, "metrics/particle/momentum/class_1/event/pt_vector_error_mean": 5.1161041259765625, "metrics/particle/momentum/class_1/event/px_bias": -0.11073759943246841, "metrics/particle/momentum/class_1/event/py_bias": -0.45631498098373413, "metrics/particle/momentum/class_1/event/pz_abs_mean": 7.980690956115723, "metrics/particle/momentum/class_1/event/pz_bias": -1.3198925256729126, "metrics/particle/momentum/class_1/event/pz_residual_rms": 15.80703353881836, "metrics/particle/momentum/class_1/event/samples": 512.0, "metrics/particle/momentum/class_1/target_samples": 16873.0, "metrics/particle/momentum/class_2/accepted/delta_r_mean": 0.04416227340698242, "metrics/particle/momentum/class_2/accepted/energy_relative_abs_mean": 0.19543619453907013, "metrics/particle/momentum/class_2/accepted/pt_relative_abs_mean": 0.19846653938293457, "metrics/particle/momentum/class_2/accepted/pt_relative_bias": -0.021844999864697456, "metrics/particle/momentum/class_2/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_2/accepted/pt_relative_residual_rms": 0.23858727514743805, "metrics/particle/momentum/class_2/accepted/pz_abs_gt10_fraction": 0.03183024004101753, "metrics/particle/momentum/class_2/accepted/pz_abs_mean": 1.563145399093628, "metrics/particle/momentum/class_2/accepted/pz_bias": -0.14526335895061493, "metrics/particle/momentum/class_2/accepted/pz_residual_rms": 3.381929874420166, "metrics/particle/momentum/class_2/accepted/samples": 1131.0, "metrics/particle/momentum/class_2/assigned/delta_r_mean": 1.0020421743392944, "metrics/particle/momentum/class_2/assigned/energy_relative_abs_mean": 0.8142555356025696, "metrics/particle/momentum/class_2/assigned/pt_relative_abs_mean": 0.8267795443534851, "metrics/particle/momentum/class_2/assigned/pt_relative_bias": 0.30556169152259827, "metrics/particle/momentum/class_2/assigned/pt_relative_gt0p5_fraction": 0.37663885951042175, "metrics/particle/momentum/class_2/assigned/pt_relative_residual_rms": 2.9813802242279053, "metrics/particle/momentum/class_2/assigned/pz_abs_gt10_fraction": 0.08581645041704178, "metrics/particle/momentum/class_2/assigned/pz_abs_mean": 3.4521124362945557, "metrics/particle/momentum/class_2/assigned/pz_bias": 0.26048457622528076, "metrics/particle/momentum/class_2/assigned/pz_residual_rms": 8.883733749389648, "metrics/particle/momentum/class_2/assigned/samples": 2517.0, "metrics/particle/momentum/class_2/efficiency": 0.3839103877544403, "metrics/particle/momentum/class_2/event/energy_bias": -8.065114974975586, "metrics/particle/momentum/class_2/event/p3_error_mean": 15.2898588180542, "metrics/particle/momentum/class_2/event/pt_vector_error_mean": 7.756397247314453, "metrics/particle/momentum/class_2/event/px_bias": -0.49135464429855347, "metrics/particle/momentum/class_2/event/py_bias": 0.12593629956245422, "metrics/particle/momentum/class_2/event/pz_abs_mean": 11.41972827911377, "metrics/particle/momentum/class_2/event/pz_bias": -0.04941178858280182, "metrics/particle/momentum/class_2/event/pz_residual_rms": 18.457263946533203, "metrics/particle/momentum/class_2/event/samples": 512.0, "metrics/particle/momentum/class_2/target_samples": 2946.0, "metrics/particle/momentum/class_3/accepted/delta_r_mean": 0.03560176119208336, "metrics/particle/momentum/class_3/accepted/energy_relative_abs_mean": 0.14175733923912048, "metrics/particle/momentum/class_3/accepted/pt_relative_abs_mean": 0.144530788064003, "metrics/particle/momentum/class_3/accepted/pt_relative_bias": 0.05494822934269905, "metrics/particle/momentum/class_3/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_3/accepted/pt_relative_residual_rms": 0.1863747090101242, "metrics/particle/momentum/class_3/accepted/pz_abs_gt10_fraction": 0.0010069571435451508, "metrics/particle/momentum/class_3/accepted/pz_abs_mean": 0.2145896703004837, "metrics/particle/momentum/class_3/accepted/pz_bias": -0.027734337374567986, "metrics/particle/momentum/class_3/accepted/pz_residual_rms": 0.7455623149871826, "metrics/particle/momentum/class_3/accepted/samples": 10924.0, "metrics/particle/momentum/class_3/assigned/delta_r_mean": 0.37036919593811035, "metrics/particle/momentum/class_3/assigned/energy_relative_abs_mean": 1.8869802951812744, "metrics/particle/momentum/class_3/assigned/pt_relative_abs_mean": 1.9163819551467896, "metrics/particle/momentum/class_3/assigned/pt_relative_bias": 1.7905480861663818, "metrics/particle/momentum/class_3/assigned/pt_relative_gt0p5_fraction": 0.18882758915424347, "metrics/particle/momentum/class_3/assigned/pt_relative_residual_rms": 17.34353256225586, "metrics/particle/momentum/class_3/assigned/pz_abs_gt10_fraction": 0.009999999776482582, "metrics/particle/momentum/class_3/assigned/pz_abs_mean": 0.678139865398407, "metrics/particle/momentum/class_3/assigned/pz_bias": -0.03585250303149223, "metrics/particle/momentum/class_3/assigned/pz_residual_rms": 4.4075493812561035, "metrics/particle/momentum/class_3/assigned/samples": 14500.0, "metrics/particle/momentum/class_3/efficiency": 0.708798348903656, "metrics/particle/momentum/class_3/event/energy_bias": 5.210264205932617, "metrics/particle/momentum/class_3/event/p3_error_mean": 9.77607250213623, "metrics/particle/momentum/class_3/event/pt_vector_error_mean": 5.302359580993652, "metrics/particle/momentum/class_3/event/px_bias": 0.7091141939163208, "metrics/particle/momentum/class_3/event/py_bias": 0.056089382618665695, "metrics/particle/momentum/class_3/event/pz_abs_mean": 7.139181137084961, "metrics/particle/momentum/class_3/event/pz_bias": -0.6686336398124695, "metrics/particle/momentum/class_3/event/pz_residual_rms": 11.801060676574707, "metrics/particle/momentum/class_3/event/samples": 512.0, "metrics/particle/momentum/class_3/target_samples": 15412.0, "metrics/particle/momentum/class_4/accepted/delta_r_mean": 0.014473620802164078, "metrics/particle/momentum/class_4/accepted/energy_relative_abs_mean": 0.05529638007283211, "metrics/particle/momentum/class_4/accepted/pt_relative_abs_mean": 0.05535990372300148, "metrics/particle/momentum/class_4/accepted/pt_relative_bias": 0.01103586982935667, "metrics/particle/momentum/class_4/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pt_relative_residual_rms": 0.10196336358785629, "metrics/particle/momentum/class_4/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/accepted/pz_abs_mean": 0.26392507553100586, "metrics/particle/momentum/class_4/accepted/pz_bias": -0.042464084923267365, "metrics/particle/momentum/class_4/accepted/pz_residual_rms": 0.9255401492118835, "metrics/particle/momentum/class_4/accepted/samples": 334.0, "metrics/particle/momentum/class_4/assigned/delta_r_mean": 0.08015123009681702, "metrics/particle/momentum/class_4/assigned/energy_relative_abs_mean": 2.097572088241577, "metrics/particle/momentum/class_4/assigned/pt_relative_abs_mean": 2.020982027053833, "metrics/particle/momentum/class_4/assigned/pt_relative_bias": 1.9661346673965454, "metrics/particle/momentum/class_4/assigned/pt_relative_gt0p5_fraction": 0.049861494451761246, "metrics/particle/momentum/class_4/assigned/pt_relative_residual_rms": 31.31650161743164, "metrics/particle/momentum/class_4/assigned/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_4/assigned/pz_abs_mean": 0.31664833426475525, "metrics/particle/momentum/class_4/assigned/pz_bias": -0.04514119401574135, "metrics/particle/momentum/class_4/assigned/pz_residual_rms": 0.9900943636894226, "metrics/particle/momentum/class_4/assigned/samples": 361.0, "metrics/particle/momentum/class_4/efficiency": 0.9226519465446472, "metrics/particle/momentum/class_4/event/energy_bias": -0.6943978667259216, "metrics/particle/momentum/class_4/event/p3_error_mean": 2.02276611328125, "metrics/particle/momentum/class_4/event/pt_vector_error_mean": 1.2559921741485596, "metrics/particle/momentum/class_4/event/px_bias": 0.026016492396593094, "metrics/particle/momentum/class_4/event/py_bias": 0.04688553139567375, "metrics/particle/momentum/class_4/event/pz_abs_mean": 1.3478732109069824, "metrics/particle/momentum/class_4/event/pz_bias": 0.36632996797561646, "metrics/particle/momentum/class_4/event/pz_residual_rms": 5.77197790145874, "metrics/particle/momentum/class_4/event/samples": 512.0, "metrics/particle/momentum/class_4/target_samples": 362.0, "metrics/particle/momentum/class_5/accepted/delta_r_mean": 0.010641931556165218, "metrics/particle/momentum/class_5/accepted/energy_relative_abs_mean": 0.02791011333465576, "metrics/particle/momentum/class_5/accepted/pt_relative_abs_mean": 0.02709018811583519, "metrics/particle/momentum/class_5/accepted/pt_relative_bias": 0.006645110435783863, "metrics/particle/momentum/class_5/accepted/pt_relative_gt0p5_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pt_relative_residual_rms": 0.08235117048025131, "metrics/particle/momentum/class_5/accepted/pz_abs_gt10_fraction": 0.0, "metrics/particle/momentum/class_5/accepted/pz_abs_mean": 0.28353792428970337, "metrics/particle/momentum/class_5/accepted/pz_bias": -0.20537659525871277, "metrics/particle/momentum/class_5/accepted/pz_residual_rms": 0.9106139540672302, "metrics/particle/momentum/class_5/accepted/samples": 181.0, "metrics/particle/momentum/class_5/assigned/delta_r_mean": 0.010781044140458107, "metrics/particle/momentum/class_5/assigned/energy_relative_abs_mean": 0.032719820737838745, "metrics/particle/momentum/class_5/assigned/pt_relative_abs_mean": 0.03189845383167267, "metrics/particle/momentum/class_5/assigned/pt_relative_bias": 0.0016514875460416079, "metrics/particle/momentum/class_5/assigned/pt_relative_gt0p5_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pt_relative_residual_rms": 0.105909064412117, "metrics/particle/momentum/class_5/assigned/pz_abs_gt10_fraction": 0.005494505632668734, "metrics/particle/momentum/class_5/assigned/pz_abs_mean": 0.5521477460861206, "metrics/particle/momentum/class_5/assigned/pz_bias": -0.4744158685207367, "metrics/particle/momentum/class_5/assigned/pz_residual_rms": 3.756188154220581, "metrics/particle/momentum/class_5/assigned/samples": 182.0, "metrics/particle/momentum/class_5/efficiency": 0.9945054650306702, "metrics/particle/momentum/class_5/event/energy_bias": -0.4037216007709503, "metrics/particle/momentum/class_5/event/p3_error_mean": 1.095041275024414, "metrics/particle/momentum/class_5/event/pt_vector_error_mean": 0.689555823802948, "metrics/particle/momentum/class_5/event/px_bias": 0.07367505878210068, "metrics/particle/momentum/class_5/event/py_bias": -0.043510034680366516, "metrics/particle/momentum/class_5/event/pz_abs_mean": 0.7327674627304077, "metrics/particle/momentum/class_5/event/pz_bias": -0.25039464235305786, "metrics/particle/momentum/class_5/event/pz_residual_rms": 3.7648401260375977, "metrics/particle/momentum/class_5/event/samples": 512.0, "metrics/particle/momentum/class_5/target_samples": 182.0}, "step_train_time": 1.5945615768432617, "step_valid_time": 3898.359181165695, "step_total_time": 3899.953742980957, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038075562446696, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07110375402705771, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 3015, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 3024, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2854, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2846, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9466003316749585, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9437830687830688, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9451896009273059, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056216931216931214, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9439469320066335, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9411375661375662, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9425401556549098, "step/clic_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9466003316749585, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0057366029579955, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08811652669400716, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 2253, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 2237, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 2094, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 2092, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.929427430093209, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9360751005811354, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9327394209354121, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06392489941886455, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9285397248113626, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9351810460438087, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.93184855233853, "step/clic_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.929427430093209, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0076719514220298, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08476646902035712, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1285, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1286, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1152, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1142, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8964980544747082, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.895800933125972, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8961493582263711, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.104199066874028, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.888715953307393, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8880248833592534, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8883702839362115, "step/clic_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8964980544747082} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/hyperparameters.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..23cfc183fc5dcadeaa2f9a99841f621ef409028c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 8639954, "dataset": "clic", "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", "model": {"type": "heptv2", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": false, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "gnnlsh": null, "attention": null, "litept": null, "hept": null, "heptv2": {"activation_checkpointing": true, "embedding_dim": 256, "width": 256, "num_convs": 6, "num_heads": 16, "dropout_ff": 0.0, "activation": "elu", "pe_type": "learned", "block_size": 128, "n_hashes": 3, "num_regions": 140, "deterministic_hashing": true, "num_w_per_dist": 10, "mlp_ratio": 4.0}}, "conv_type": "heptv2", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 50000, "patience": 50000, "checkpoint_freq": 1000, "val_freq": 1000, "num_workers": 2, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 128, "dtype": "bfloat16", "lr": 0.001, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "cosinedecay", "lr_schedule_config": {}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 4, "validation_linear_probe": true, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 100, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"clic": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"clic": {"physical": {"batch_size": 1, "weight": null, "samples": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"clic_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_ww_fullhad_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}, "clic_edm_qq_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["clic_edm_ttbar_pf", "clic_edm_ww_fullhad_pf", "clic_edm_qq_pf"], "ntrain": null, "nvalid": 512, "ntest": 512, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/model_kwargs.pkl b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..946793d11f44693566a644afe9f0cc28adbe934f --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca7e4d3eb5002a66d9e0314201360d711a541e8b601ce0ec81f5e8f7c3e6da2 +size 4870 diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/particleflow_spec.yaml b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..764890baecded6715d37cf35f42c4079f6e00e2c --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/particleflow_spec.yaml @@ -0,0 +1,2551 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.0.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.2.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.0 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.0 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.0.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.2.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.0 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.0 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id019 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id020 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id019 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id020 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5b83660453c82a54154f1ef6dcbf597a660bfef6 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..8861d3d973f91e6adb8621cc3cf765f4131ca37a Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..58e6a8f590b422d82141a7867f031e3c7415a1dc Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..594e46a414d02fef6be5cf603f6b01e571909fc3 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3183b3ab49c9ddcd2afd027700c1d2dad7f7fb73 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..dd5bdfd16a94680499de60879acf300688c49c84 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5e3a87d7cde963fae9799a34f47918bfa04a7e79 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..ec31016338986b73f6ce7b84d9a36a1f796d06ab Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..169d8621d8827c719066868d0e0ba4112c419549 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..9e2929f3d5e359e5b927336ff466f4219d9f30ba Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..57569526fb651dd31bb91a814b4753f138884832 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..da41137f4fc67ff40e43a54c57f60c4e85c8c463 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_qq_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d12bbc4452ba30a4bd81aa5867f3ffcca734b1e5 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..af26fc30098e57a723695b9b367c05501696b471 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6a38222489d58d2225943fcb54942662e99da0b3 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..6e015bb3ae0484d181ad2c2f9d616327e341a87e Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..41b675c9f3ff28fad67f7fbf2ef06aa76ff6f4bd Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..ba6951a6a7250c2881e8544285625864a665bd5b Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5b9a9e9e10d2f1911f7f75f4255457fae0d76ab5 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..4ed9c06997a0af7da2a2b24090564626c06156ea Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c112243def3a3d36dbc6e635b0d05f9d9a1a555b Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..6e6b9454675c848fc74e5808fc1a8960812afa57 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..10a3f6f1d5de371100788e933a8c7e4ea831a23d Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..26eb917e7232c3ee93ab70c8f967ca2199d6a33c Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ttbar_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3a09fda49b74e9ba7568288be239adda65e18df9 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..44763b4a87bbe572625c0e5200dc172cd82cfae5 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_eta.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fa2f7662792c613adfb161bf6cb15fc995e5bc9a Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..92be8c3b9bcc72256a954d4fb3536f3f34e89210 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9c0162bd1bac328b5c2e6518ddd6918e557293da Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..ab4b68ca5269e57610352d4e56eb4d3addf6cba0 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_pt_log.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..638781a5d9624ad5bdfb6c772060c429670fded2 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..b5f794e103b539ab6e04b9196712d985260816dc Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dd3029b102928446e978df04c6159814b4c468a9 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..64b31c29cb13b5ee44072199bf2ce33e74cd2b64 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/jet_res_vstarget.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..05b64736317d94dad942727d48ba87900ea5ef6c Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.pdf differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b85dd7603ef820f1dd1bf9900a8e17ef651e0d78 Binary files /dev/null and b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/plots_step_50000/clic_edm_ww_fullhad_pf/num_elements.png differ diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/train/events.out.tfevents.1793178753.nid005803.1218.0 b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/train/events.out.tfevents.1793178753.nid005803.1218.0 new file mode 100644 index 0000000000000000000000000000000000000000..0ffff9f9ede5a1116c21331db12fc3a9582e55ad --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/train/events.out.tfevents.1793178753.nid005803.1218.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc2a84f86e72c8311be083c54ec242a8adbe0bc6d2ed3f977bdb53d27a96188 +size 16258558 diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/valid/events.out.tfevents.1793178753.nid005803.1218.1 b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/valid/events.out.tfevents.1793178753.nid005803.1218.1 new file mode 100644 index 0000000000000000000000000000000000000000..5168d119923c4fab1e00b27732d4f98deb300390 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/runs/valid/events.out.tfevents.1793178753.nid005803.1218.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8fa0ea6cd853d9eb82f129f91e1a70a6c2257d3b324549db3f7358a4b48ad7 +size 28576120 diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/scenario-manifest.json b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..54651da1c1bf45690fd04d28b58473500b3e23fc --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/scenario-manifest.json @@ -0,0 +1,846 @@ +{ + "scenario": { + "name": "clic_cld_backbone_benchmark", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld_pf_attention": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": {} + }, + "cld_pf_heptv2": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "cld_hits_attention": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "cld_hits_heptv2": { + "model_name": "pyg-cld-hits-set-v1", + "production_name": null, + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_pf_attention": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": {} + }, + "clic_pf_heptv2": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": 1, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true + } + }, + "clic_hits_attention": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + }, + "clic_hits_heptv2": { + "model_name": "pyg-clic-hits-set-v1", + "production_name": "clic", + "gpus": null, + "overrides": { + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "model.set_decoder.num_slots": 256, + "model.set_decoder.num_layers": 4, + "model.set_decoder.num_heads": 8, + "model.set_decoder.attention_mode": "global-flash", + "model.set_decoder.query_init": "input-conditioned", + "model.set_decoder.local_attention_radius": 0.4, + "model.set_decoder.tracker_query_fraction": 0.6, + "model.set_decoder.proposal_mode": "grid-diverse", + "model.set_decoder.proposal_grid_size": 0.1, + "model.set_decoder.use_aggregate_anchors": true, + "model.set_decoder.aggregate_grid_size": 0.1, + "model.set_decoder.presence_threshold": 0.5, + "model.set_decoder.no_object_weight": 1.0, + "model.set_decoder.cardinality_loss_weight": 0.05, + "model.set_decoder.auxiliary_loss_weight": 0.25, + "model.set_decoder.matcher.presence": 1.0, + "model.set_decoder.matcher.pid": 1.0, + "model.set_decoder.matcher.geometry": 2.0, + "model.set_decoder.matcher.pt": 1.0, + "model.set_decoder.matcher.energy": 0.0, + "model.set_decoder.matcher.dr_scale": 0.1, + "model.set_decoder.matcher.log_pt_scale": 0.6931471805599453, + "model.set_decoder.matcher.log_energy_scale": 0.6931471805599453, + "model.set_decoder.hit_grouping": true, + "model.set_decoder.hit_grouping_dim": 32, + "model.set_decoder.hit_grouping_chunk_size": 1024, + "model.set_decoder.hit_grouping_loss_weight": 0.1, + "model.set_decoder.hit_grouping_background_weight": 0.1, + "model.set_decoder.hit_grouping_detector_balanced": false, + "model.set_decoder.hit_grouping_sparse": false, + "model.set_decoder.hit_grouping_tracker_particle_balanced": false, + "model.set_decoder.tracker_momentum_readout": true, + "model.set_decoder.tracker_group_readout": false, + "model.set_decoder.tracker_fit_correction": false, + "model.set_decoder.tracker_overlap_loss_weight": 0.0, + "model.set_decoder.tracker_geometry_loss_weight": 0.0, + "model.set_decoder.tracker_curvature_loss_weight": 0.0, + "model.set_decoder.event_consistency_loss_weight": 0.0, + "model.hit_feature_engineering.enabled": true, + "model.hit_feature_engineering.geometry": true, + "model.hit_feature_engineering.calorimeter_neighborhood": true, + "model.hit_feature_engineering.tracker_neighborhood": true, + "model.hit_feature_engineering.tracker_momentum": false, + "model.hit_feature_engineering.tracker_doublets": false + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 128, + "parameters": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null + } + }, + "common_overrides": { + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true + }, + "allowed_variant_differences": [ + "conv_type", + "data_dir", + "dataset", + "elemtypes_nonzero", + "enabled_test_datasets", + "hyperparameters.lr", + "input_dim", + "model.type", + "model.attention", + "model.heptv2", + "model.binary_classification_focal_gamma", + "model.hit_feature_engineering.tracker_neighborhood", + "model.output_mode", + "model.set_decoder", + "train_dataset", + "valid_dataset", + "test_dataset" + ] + }, + "platform": { + "name": "lumi_mi250x", + "gpus": 8, + "python_command": [ + "python3" + ], + "data_dir": { + "cld": "/scratch/project_465001293/joosep/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "cms_run3": "/scratch/project_465001293/joosep/mlpf/cms/2.8.0/" + }, + "experiments_dir": "/scratch/project_465001293/patajoos/particleflow/experiments", + "environment": { + "MIOPEN_USER_DB_PATH": "/tmp/patajoos-23239268-miopen-cache", + "MIOPEN_CUSTOM_CACHE_DIR": "/tmp/patajoos-23239268-miopen-cache", + "ROCM_PATH": "/opt/rocm", + "KERAS_BACKEND": "torch", + "OMP_NUM_THREADS": "4", + "PYTHONUNBUFFERED": "1", + "NCCL_SOCKET_IFNAME": "hsn", + "NCCL_NET_GDR_LEVEL": "3" + }, + "runtime_overrides": { + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true + }, + "slurm": { + "partition": "standard-g", + "constraint": null, + "account": "project_465001293", + "time": "2-00:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 32, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "480G", + "memory_per_gpu": null, + "no_requeue": true + } + }, + "job": { + "scenario_name": "clic_cld_backbone_benchmark", + "platform_name": "lumi_mi250x", + "variant_name": "clic_pf_heptv2", + "model_name": "pyg-clic-v1", + "production_name": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "seed": 12345, + "global_batch_size": 128, + "per_gpu_batch_size": 128, + "gpu_batch_multiplier": 128, + "settings": { + "optimizer": "lamb", + "lr": 0.001, + "lr_schedule": "cosinedecay", + "num_steps": 50000, + "patience": 50000, + "val_freq": 1000, + "checkpoint_freq": 1000, + "nvalid": 512, + "ntest": 512, + "sampler_mode": "interleaved-shards", + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "pad_to_multiple_elements": 128, + "make_plots": true, + "load": null, + "model.trainable": "all", + "model.task_queries": false, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "model.attention.num_convs": 6, + "model.attention.num_heads": 16, + "model.attention.head_dim": 16, + "model.attention.use_jagged_attention": true, + "compile": false, + "dtype": "bfloat16", + "num_workers": 2, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": true, + "model.type": "heptv2", + "model.attention": null, + "model.heptv2.embedding_dim": 256, + "model.heptv2.width": 256, + "model.heptv2.num_convs": 6, + "model.heptv2.num_heads": 16, + "model.heptv2.dropout_ff": 0.0, + "model.heptv2.block_size": 128, + "model.heptv2.deterministic_hashing": true, + "model.heptv2.activation_checkpointing": true, + "seed": 12345, + "gpu_batch_multiplier": 128 + } + }, + "resolved_config": { + "dataset": "clic", + "data_dir": "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "model": { + "type": "heptv2", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": false, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "gnnlsh": null, + "attention": null, + "litept": null, + "hept": null, + "heptv2": { + "activation_checkpointing": true, + "embedding_dim": 256, + "width": 256, + "num_convs": 6, + "num_heads": 16, + "dropout_ff": 0.0, + "activation": "elu", + "pe_type": "learned", + "block_size": 128, + "n_hashes": 3, + "num_regions": 140, + "deterministic_hashing": true, + "num_w_per_dist": 10, + "mlp_ratio": 4.0 + } + }, + "conv_type": "heptv2", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 50000, + "patience": 50000, + "checkpoint_freq": 1000, + "val_freq": 1000, + "num_workers": 2, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 128, + "dtype": "bfloat16", + "lr": 0.001, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "cosinedecay", + "lr_schedule_config": {}, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 4, + "validation_linear_probe": true, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 100, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "clic": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "clic": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "clic_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_ww_fullhad_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + }, + "clic_edm_qq_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "clic_edm_ttbar_pf", + "clic_edm_ww_fullhad_pf", + "clic_edm_qq_pf" + ], + "ntrain": null, + "nvalid": 512, + "ntest": 512, + "data_config": null, + "gpus": 1 + }, + "command": [ + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-clic-v1", + "--production-name", + "clic", + "--data-dir", + "/scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "--experiment-dir", + "/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.001", + "--lr_schedule", + "cosinedecay", + "--num_steps", + "50000", + "--patience", + "50000", + "--val_freq", + "1000", + "--checkpoint_freq", + "1000", + "--nvalid", + "512", + "--ntest", + "512", + "--sampler_mode", + "interleaved-shards", + "--validation_diagnostics_batches", + "4", + "--validation_linear_probe", + "true", + "--validation_linear_probe_max_events", + "16", + "--pad_to_multiple_elements", + "128", + "--model.trainable", + "all", + "--model.task_queries", + "false", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--model.attention.num_convs", + "6", + "--model.attention.num_heads", + "16", + "--model.attention.head_dim", + "16", + "--model.attention.use_jagged_attention", + "true", + "--num_workers", + "2", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "true", + "--model.type", + "heptv2", + "--model.attention", + "null", + "--model.heptv2.embedding_dim", + "256", + "--model.heptv2.width", + "256", + "--model.heptv2.num_convs", + "6", + "--model.heptv2.num_heads", + "16", + "--model.heptv2.dropout_ff", + "0.0", + "--model.heptv2.block_size", + "128", + "--model.heptv2.deterministic_hashing", + "true", + "--model.heptv2.activation_checkpointing", + "true", + "--seed", + "12345", + "--gpu_batch_multiplier", + "128" + ], + "git_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc" +} \ No newline at end of file diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train-config.yaml b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b52ff6463b6a5824632cea34490d7fff579bb1d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train-config.yaml @@ -0,0 +1,276 @@ +backend: pytorch +batch_size: 1 +checkpoint_freq: 1000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: heptv2 +data_config: null +data_dir: /scratch/project_465001293/joosep/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds +dataset: clic +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- clic_edm_ttbar_pf +- clic_edm_ww_fullhad_pf +- clic_edm_qq_pf +gpu_batch_multiplier: 128 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.001 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: cosinedecay +lr_schedule_config: {} +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: null + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + gnnlsh: null + hept: null + heptv2: + activation: elu + activation_checkpointing: true + block_size: 128 + deterministic_hashing: true + dropout_ff: 0.0 + embedding_dim: 256 + mlp_ratio: 4.0 + n_hashes: 3 + num_convs: 6 + num_heads: 16 + num_regions: 140 + num_w_per_dist: 10 + pe_type: learned + width: 256 + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: false + trainable: all + type: heptv2 +ntest: 512 +ntrain: null +num_classes: 6 +num_steps: 50000 +num_workers: 2 +nvalid: 512 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 50000 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 100 +test: true +test_dataset: + clic_edm_qq_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + clic: + physical: + batch_size: 1 + samples: + clic_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 1000 +valid_dataset: + clic: + physical: + batch_size: 1 + samples: + clic_edm_qq_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic_edm_ww_fullhad_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 4 +validation_linear_probe: true +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3968fa0d940392fa5637ef243009d8ed4cfce11d --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log @@ -0,0 +1,13931 @@ +rank0 [2026-10-28 11:12:24,877] INFO : Will use single-gpu: AMD Instinct MI250X +rank0 [2026-10-28 11:12:24,878] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-10-28 11:12:24,884] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,885] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,885] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,885] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,886] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,922] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,923] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,923] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,923] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,976] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,983] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,991] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,998] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:25,006] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:25,013] INFO : Backbone initialization took 0.09s +rank0 [2026-10-28 11:12:25,014] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:25,014] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:25,018] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:25,019] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:25,019] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:25,019] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:25,020] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:25,020] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:25,021] INFO : Total MLPF parameters: 8639954 +rank0 [2026-10-28 11:12:25,021] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:25,021] INFO : Instantiated model +rank0 [2026-10-28 11:12:25,021] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,241] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,242] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,356] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +205 nn_energy.nn.1.0.bias 256 0 +206 nn_energy.nn.1.2.weight 256 0 +207 nn_energy.nn.1.2.bias 256 0 +208 nn_energy.nn.1.4.weight 256 0 +209 nn_energy.nn.1.4.bias 1 0 + +[210 rows x 3 columns] +rank0 [2026-10-28 11:12:25,356] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,357] INFO : Trainable parameters: 8639954 +rank0 [2026-10-28 11:12:25,357] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:25,358] INFO : Total parameters: 8663222 +rank0 [2026-10-28 11:12:25,360] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.regions 0 96 + backbone.0.attn.out_linear.weight 65536 0 + backbone.0.attn.out_linear.bias 256 0 + backbone.0.attn.e2lsh.alpha 0 864 + backbone.0.attn.e2lsh.beta 0 3 + backbone.0.attn.e2lsh_new.alpha 0 96 + backbone.0.attn.e2lsh_new.beta 0 3 + backbone.0.w_q.weight 65536 0 + backbone.0.w_k.weight 65536 0 + backbone.0.w_v.weight 65536 0 + backbone.0.w_rpe.weight 0 2560 + backbone.0.w_rpe.bias 0 256 +backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + backbone.0.pe_func.position_embedding_head.0.bias 1024 0 +backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + backbone.0.pe_func.position_embedding_head.1.bias 1024 0 +backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + backbone.0.pe_func.position_embedding_head.3.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm2.weight 256 0 + backbone.0.q_norm.weight 16 0 + backbone.0.k_norm.weight 16 0 + backbone.0.ff.gate_proj.weight 262144 0 + backbone.0.ff.up_proj.weight 262144 0 + backbone.0.ff.down_proj.weight 262144 0 + backbone.1.regions 0 96 + backbone.1.attn.out_linear.weight 65536 0 + backbone.1.attn.out_linear.bias 256 0 + backbone.1.attn.e2lsh.alpha 0 864 + backbone.1.attn.e2lsh.beta 0 3 + backbone.1.attn.e2lsh_new.alpha 0 96 + backbone.1.attn.e2lsh_new.beta 0 3 + backbone.1.w_q.weight 65536 0 + backbone.1.w_k.weight 65536 0 + backbone.1.w_v.weight 65536 0 + backbone.1.w_rpe.weight 0 2560 + backbone.1.w_rpe.bias 0 256 +backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + backbone.1.pe_func.position_embedding_head.0.bias 1024 0 +backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + backbone.1.pe_func.position_embedding_head.1.bias 1024 0 +backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + backbone.1.pe_func.position_embedding_head.3.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm2.weight 256 0 + backbone.1.q_norm.weight 16 0 + backbone.1.k_norm.weight 16 0 + backbone.1.ff.gate_proj.weight 262144 0 + backbone.1.ff.up_proj.weight 262144 0 + backbone.1.ff.down_proj.weight 262144 0 + backbone.2.regions 0 96 + backbone.2.attn.out_linear.weight 65536 0 + backbone.2.attn.out_linear.bias 256 0 + backbone.2.attn.e2lsh.alpha 0 864 + backbone.2.attn.e2lsh.beta 0 3 + backbone.2.attn.e2lsh_new.alpha 0 96 + backbone.2.attn.e2lsh_new.beta 0 3 + backbone.2.w_q.weight 65536 0 + backbone.2.w_k.weight 65536 0 + backbone.2.w_v.weight 65536 0 + backbone.2.w_rpe.weight 0 2560 + backbone.2.w_rpe.bias 0 256 +backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + backbone.2.pe_func.position_embedding_head.0.bias 1024 0 +backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + backbone.2.pe_func.position_embedding_head.1.bias 1024 0 +backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + backbone.2.pe_func.position_embedding_head.3.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm2.weight 256 0 + backbone.2.q_norm.weight 16 0 + backbone.2.k_norm.weight 16 0 + backbone.2.ff.gate_proj.weight 262144 0 + backbone.2.ff.up_proj.weight 262144 0 + backbone.2.ff.down_proj.weight 262144 0 + backbone.3.regions 0 96 + backbone.3.attn.out_linear.weight 65536 0 + backbone.3.attn.out_linear.bias 256 0 + backbone.3.attn.e2lsh.alpha 0 864 + backbone.3.attn.e2lsh.beta 0 3 + backbone.3.attn.e2lsh_new.alpha 0 96 + backbone.3.attn.e2lsh_new.beta 0 3 + backbone.3.w_q.weight 65536 0 + backbone.3.w_k.weight 65536 0 + backbone.3.w_v.weight 65536 0 + backbone.3.w_rpe.weight 0 2560 + backbone.3.w_rpe.bias 0 256 +backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + backbone.3.pe_func.position_embedding_head.0.bias 1024 0 +backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + backbone.3.pe_func.position_embedding_head.1.bias 1024 0 +backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + backbone.3.pe_func.position_embedding_head.3.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm2.weight 256 0 + backbone.3.q_norm.weight 16 0 + backbone.3.k_norm.weight 16 0 + backbone.3.ff.gate_proj.weight 262144 0 + backbone.3.ff.up_proj.weight 262144 0 + backbone.3.ff.down_proj.weight 262144 0 + backbone.4.regions 0 96 + backbone.4.attn.out_linear.weight 65536 0 + backbone.4.attn.out_linear.bias 256 0 + backbone.4.attn.e2lsh.alpha 0 864 + backbone.4.attn.e2lsh.beta 0 3 + backbone.4.attn.e2lsh_new.alpha 0 96 + backbone.4.attn.e2lsh_new.beta 0 3 + backbone.4.w_q.weight 65536 0 + backbone.4.w_k.weight 65536 0 + backbone.4.w_v.weight 65536 0 + backbone.4.w_rpe.weight 0 2560 + backbone.4.w_rpe.bias 0 256 +backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + backbone.4.pe_func.position_embedding_head.0.bias 1024 0 +backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + backbone.4.pe_func.position_embedding_head.1.bias 1024 0 +backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + backbone.4.pe_func.position_embedding_head.3.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm2.weight 256 0 + backbone.4.q_norm.weight 16 0 + backbone.4.k_norm.weight 16 0 + backbone.4.ff.gate_proj.weight 262144 0 + backbone.4.ff.up_proj.weight 262144 0 + backbone.4.ff.down_proj.weight 262144 0 + backbone.5.regions 0 96 + backbone.5.attn.out_linear.weight 65536 0 + backbone.5.attn.out_linear.bias 256 0 + backbone.5.attn.e2lsh.alpha 0 864 + backbone.5.attn.e2lsh.beta 0 3 + backbone.5.attn.e2lsh_new.alpha 0 96 + backbone.5.attn.e2lsh_new.beta 0 3 + backbone.5.w_q.weight 65536 0 + backbone.5.w_k.weight 65536 0 + backbone.5.w_v.weight 65536 0 + backbone.5.w_rpe.weight 0 2560 + backbone.5.w_rpe.bias 0 256 +backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + backbone.5.pe_func.position_embedding_head.0.bias 1024 0 +backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + backbone.5.pe_func.position_embedding_head.1.bias 1024 0 +backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + backbone.5.pe_func.position_embedding_head.3.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm2.weight 256 0 + backbone.5.q_norm.weight 16 0 + backbone.5.k_norm.weight 16 0 + backbone.5.ff.gate_proj.weight 262144 0 + backbone.5.ff.up_proj.weight 262144 0 + backbone.5.ff.down_proj.weight 262144 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,364] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554 +rank0 [2026-10-28 11:12:25,364] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554 +rank0 [2026-10-28 11:12:25,364] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,364] INFO : sample=clic_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,313] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,354] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,395] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,417] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,473] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,512] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,549] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,590] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,614] INFO : train_dataset: clic_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,614] INFO : sample=clic_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,650] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,679] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,703] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,745] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,783] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,814] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,842] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,877] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,900] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,935] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,935] INFO : sample=clic_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,973] INFO : train_dataset: clic_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,011] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,082] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,102] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,123] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,172] INFO : train_dataset: clic_edm_qq_pf, 89996 +rank0 [2026-10-28 11:12:33,198] INFO : train_dataset: clic_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:33,229] INFO : train_dataset: clic_edm_qq_pf, 89994 +rank0 [2026-10-28 11:12:33,263] INFO : train_dataset: clic_edm_qq_pf, 89995 +rank0 [2026-10-28 11:12:33,278] INFO : train_dataset: clic_edm_qq_pf, 90099 +rank0 [2026-10-28 11:12:33,278] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,278] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,278] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,278] INFO : sample=clic_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,290] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,296] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,302] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,307] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,313] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,319] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,325] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,331] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,337] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,337] INFO : sample=clic_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,342] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,347] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,352] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,357] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,363] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,368] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,373] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,378] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,383] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : sample=clic_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,393] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,398] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,403] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,408] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,413] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,418] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,423] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,428] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,434] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,438] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,439] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,469] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,470] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:12:33,470] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,470] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,476] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,494] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,747] INFO : Created train_iterator +rank0 [2026-10-28 11:13:44,783] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 4.85045928478241, 'Classification': 0.11577398337423801, 'Regression_pt': 0.32241056829690934, 'Regression_eta': 0.09871801823377609, 'Regression_sin_phi': 0.0849964850768447, 'Regression_cos_phi': 0.0726868094317615, 'Regression_energy': 0.3507564795017242}, weights={'Classification_binary': 1.0, 'Classification': 41.89593505859375, 'Regression_pt': 15.044355392456055, 'Regression_eta': 49.13448715209961, 'Regression_sin_phi': 57.066585540771484, 'Regression_cos_phi': 66.73094177246094, 'Regression_energy': 13.828566551208496} +rank0 [2026-10-28 11:13:45,975] INFO : Step 100/50000 rank0 | Train Loss: 5.1883 | Train Loss Unweighted: 5.1883 | Train Loss Components: Classification: 0.0537 | Classification_binary: 4.4025 | Regression_cos_phi: 0.0422 | Regression_energy: 0.3073 | Regression_eta: 0.0481 | Regression_pt: 0.3016 | Regression_sin_phi: 0.0330 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 4.4025 | Regression_cos_phi: 0.0422 | Regression_energy: 0.3073 | Regression_eta: 0.0481 | Regression_pt: 0.3016 | Regression_sin_phi: 0.0330 | LR: 1.00e-03 | DataLoad Time: 0.0021s | Model Forward Time: 1.6084s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0516 | max_backward: 0.2038 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.4370 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2956.1 | uss_MB: 2130.6 | child_rss_MB: 18335.3 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:13:46,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:25,886] INFO : Step 200/50000 rank0 | Train Loss: 16.9962 | Train Loss Unweighted: 4.8663 | Train Loss Components: Classification: 0.0438 | Classification_binary: 4.2706 | Regression_cos_phi: 0.0230 | Regression_energy: 0.2439 | Regression_eta: 0.0279 | Regression_pt: 0.2395 | Regression_sin_phi: 0.0177 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.8349 | Classification_binary: 4.2706 | Regression_cos_phi: 1.5358 | Regression_energy: 3.3722 | Regression_eta: 1.3692 | Regression_pt: 3.6034 | Regression_sin_phi: 1.0102 | LR: 1.00e-03 | DataLoad Time: 0.0025s | Model Forward Time: 1.3899s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1847 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.2732 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2955.8 | uss_MB: 2135.4 | child_rss_MB: 4121.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:14:26,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:05,302] INFO : Step 300/50000 rank0 | Train Loss: 16.2052 | Train Loss Unweighted: 4.6492 | Train Loss Components: Classification: 0.0407 | Classification_binary: 4.0329 | Regression_cos_phi: 0.0158 | Regression_energy: 0.2619 | Regression_eta: 0.0186 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0160 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.7067 | Classification_binary: 4.0329 | Regression_cos_phi: 1.0536 | Regression_energy: 3.6220 | Regression_eta: 0.9162 | Regression_pt: 3.9587 | Regression_sin_phi: 0.9151 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4383s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0772 | max_loss: 0.0029 | max_backward: 0.3085 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.3489 | max_tensorboard_flush: 0.0080 | Memory: rss_MB: 2955.8 | uss_MB: 2135.4 | child_rss_MB: 4145.8 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:05,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:44,934] INFO : Step 400/50000 rank0 | Train Loss: 13.6138 | Train Loss Unweighted: 4.5148 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.0488 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1931 | Regression_eta: 0.0148 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.7460 | Classification_binary: 4.0488 | Regression_cos_phi: 0.8709 | Regression_energy: 2.6705 | Regression_eta: 0.7272 | Regression_pt: 2.8833 | Regression_sin_phi: 0.6671 | LR: 1.00e-03 | DataLoad Time: 0.0018s | Model Forward Time: 1.5434s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0032 | max_backward: 0.3822 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.3636 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2957.5 | uss_MB: 2137.5 | child_rss_MB: 4115.6 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 11:15:45,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:25,370] INFO : Step 500/50000 rank0 | Train Loss: 14.5518 | Train Loss Unweighted: 4.4666 | Train Loss Components: Classification: 0.0392 | Classification_binary: 3.9419 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2201 | Regression_eta: 0.0197 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6435 | Classification_binary: 3.9419 | Regression_cos_phi: 0.8466 | Regression_energy: 3.0437 | Regression_eta: 0.9661 | Regression_pt: 3.2879 | Regression_sin_phi: 0.8220 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4333s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1048 | max_forward: 0.0777 | max_loss: 0.0029 | max_backward: 0.2038 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.2531 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2955.2 | uss_MB: 2135.4 | child_rss_MB: 4113.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:16:25,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:05,330] INFO : Step 600/50000 rank0 | Train Loss: 15.3389 | Train Loss Unweighted: 4.6861 | Train Loss Components: Classification: 0.0405 | Classification_binary: 4.1250 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2384 | Regression_eta: 0.0204 | Regression_pt: 0.2344 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6955 | Classification_binary: 4.1250 | Regression_cos_phi: 0.8952 | Regression_energy: 3.2969 | Regression_eta: 1.0005 | Regression_pt: 3.5257 | Regression_sin_phi: 0.8000 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.4723s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3644 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2892.7 | uss_MB: 2073.6 | child_rss_MB: 4109.0 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:17:05,543] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:44,967] INFO : Step 700/50000 rank0 | Train Loss: 13.9769 | Train Loss Unweighted: 4.4707 | Train Loss Components: Classification: 0.0397 | Classification_binary: 3.9779 | Regression_cos_phi: 0.0138 | Regression_energy: 0.2076 | Regression_eta: 0.0140 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6613 | Classification_binary: 3.9779 | Regression_cos_phi: 0.9199 | Regression_energy: 2.8704 | Regression_eta: 0.6874 | Regression_pt: 3.0670 | Regression_sin_phi: 0.7929 | LR: 1.00e-03 | DataLoad Time: 0.0022s | Model Forward Time: 1.4228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0703 | max_backward: 0.2035 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.3675 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2892.6 | uss_MB: 2074.6 | child_rss_MB: 4110.5 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:17:45,181] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:25,118] INFO : Step 800/50000 rank0 | Train Loss: 13.4356 | Train Loss Unweighted: 4.3883 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.9055 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2047 | Regression_eta: 0.0138 | Regression_pt: 0.2044 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5736 | Classification_binary: 3.9055 | Regression_cos_phi: 0.7061 | Regression_energy: 2.8310 | Regression_eta: 0.6765 | Regression_pt: 3.0749 | Regression_sin_phi: 0.6678 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4582s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3107 | max_optimizer_step: 0.1036 | max_tensorboard_logging: 0.3634 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 2895.4 | uss_MB: 2076.2 | child_rss_MB: 4113.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:18:25,333] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:05,318] INFO : Step 900/50000 rank0 | Train Loss: 13.7821 | Train Loss Unweighted: 4.3842 | Train Loss Components: Classification: 0.0391 | Classification_binary: 3.8885 | Regression_cos_phi: 0.0107 | Regression_energy: 0.2093 | Regression_eta: 0.0150 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6371 | Classification_binary: 3.8885 | Regression_cos_phi: 0.7142 | Regression_energy: 2.8946 | Regression_eta: 0.7358 | Regression_pt: 3.1292 | Regression_sin_phi: 0.7827 | LR: 9.99e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.3648s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3086 | max_optimizer_step: 0.1025 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 2895.4 | uss_MB: 2076.2 | child_rss_MB: 4112.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:19:05,530] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:44,801] INFO : Step 1000/50000 rank0 | Train Loss: 13.0955 | Train Loss Unweighted: 4.3589 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.8991 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1933 | Regression_eta: 0.0146 | Regression_pt: 0.1936 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4931 | Classification_binary: 3.8991 | Regression_cos_phi: 0.7407 | Regression_energy: 2.6735 | Regression_eta: 0.7169 | Regression_pt: 2.9129 | Regression_sin_phi: 0.6594 | LR: 9.99e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3523s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3049 | max_optimizer_step: 0.1051 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2895.5 | uss_MB: 2076.2 | child_rss_MB: 4109.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:19:45,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:45,032] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:19:45,208] INFO : Running validation on rank0 +rank0 [2026-10-28 11:19:45,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:19:45,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:19:47,539] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:19:47,723] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:19:47,821] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:19:47,880] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:19:47,940] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:19:47,989] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:19:48,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:19:48,110] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:19:48,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:19:48,260] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:19:48,358] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:19:48,419] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:19:48,482] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:19:48,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:19:48,595] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:19:48,655] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:19:48,713] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:19:48,717] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:19:48,720] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:19:48,723] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:19:48,723] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:19:48,726] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:19:48,729] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:19:48,732] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:19:48,735] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:19:48,738] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:19:48,739] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:19:48,741] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:19:48,744] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:19:48,747] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:19:48,750] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:19:48,753] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:19:56,086] INFO : VALIDATION | Step=1000/50000 | Train Loss=13.0955 | Valid Loss=13.7521 | Train Loss Unweighted=4.3589 | Valid Loss Unweighted=4.3445 | Stale=0 | ETA=361.5m +rank0 [2026-10-28 11:19:56,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:19:56,160] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:19:56,169] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:19:56,319] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:19:59,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:00,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:00,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:01,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:01,445] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-10-28 11:20:01,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:01,506] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:01,509] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:01,652] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:03,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:04,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:04,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:05,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:05,564] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:05,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:05,621] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:05,624] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:20:05,765] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:07,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:08,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:08,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:09,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:09,689] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:09,697] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:09,952] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:10,575] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:12,446] INFO : Plotted jets +rank0 [2026-10-28 11:20:13,537] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:13,569] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:13,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:14,169] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:15,540] INFO : Plotted jets +rank0 [2026-10-28 11:20:16,649] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:16,682] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:16,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:17,296] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:18,664] INFO : Plotted jets +rank0 [2026-10-28 11:20:19,752] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:00,512] INFO : Step 1100/50000 rank0 | Train Loss: 13.6489 | Train Loss Unweighted: 4.2287 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.7268 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2127 | Regression_eta: 0.0159 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5704 | Classification_binary: 3.7268 | Regression_cos_phi: 0.8141 | Regression_energy: 2.9414 | Regression_eta: 0.7824 | Regression_pt: 3.2019 | Regression_sin_phi: 0.6119 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5259s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0867 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3210.7 | uss_MB: 2399.3 | child_rss_MB: 17792.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:21:00,737] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:39,790] INFO : Step 1200/50000 rank0 | Train Loss: 13.7197 | Train Loss Unweighted: 4.2789 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7803 | Regression_cos_phi: 0.0138 | Regression_energy: 0.2121 | Regression_eta: 0.0126 | Regression_pt: 0.2105 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4551 | Classification_binary: 3.7803 | Regression_cos_phi: 0.9219 | Regression_energy: 2.9335 | Regression_eta: 0.6202 | Regression_pt: 3.1672 | Regression_sin_phi: 0.8415 | LR: 9.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4191s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1852 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3210.5 | uss_MB: 2403.9 | child_rss_MB: 4105.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:21:40,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:19,313] INFO : Step 1300/50000 rank0 | Train Loss: 13.6193 | Train Loss Unweighted: 4.2909 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7996 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2108 | Regression_eta: 0.0134 | Regression_pt: 0.2038 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5366 | Classification_binary: 3.7996 | Regression_cos_phi: 0.8593 | Regression_energy: 2.9146 | Regression_eta: 0.6600 | Regression_pt: 3.0653 | Regression_sin_phi: 0.7839 | LR: 9.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2495 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.4122 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3210.5 | uss_MB: 2403.8 | child_rss_MB: 4103.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:22:19,525] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:59,245] INFO : Step 1400/50000 rank0 | Train Loss: 12.7212 | Train Loss Unweighted: 4.1945 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.7320 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1979 | Regression_eta: 0.0101 | Regression_pt: 0.1993 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3757 | Classification_binary: 3.7320 | Regression_cos_phi: 0.7072 | Regression_energy: 2.7364 | Regression_eta: 0.4961 | Regression_pt: 2.9987 | Regression_sin_phi: 0.6751 | LR: 9.98e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4799s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3079 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3164 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3209.1 | uss_MB: 2402.5 | child_rss_MB: 4107.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:22:59,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:39,013] INFO : Step 1500/50000 rank0 | Train Loss: 12.6088 | Train Loss Unweighted: 4.0521 | Train Loss Components: Classification: 0.0348 | Classification_binary: 3.6019 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1892 | Regression_eta: 0.0135 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4581 | Classification_binary: 3.6019 | Regression_cos_phi: 0.7975 | Regression_energy: 2.6160 | Regression_eta: 0.6626 | Regression_pt: 2.8576 | Regression_sin_phi: 0.6151 | LR: 9.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4508s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1838 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3208.4 | uss_MB: 2402.5 | child_rss_MB: 4103.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:23:39,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:18,807] INFO : Step 1600/50000 rank0 | Train Loss: 13.0472 | Train Loss Unweighted: 4.0810 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.6044 | Regression_cos_phi: 0.0146 | Regression_energy: 0.2052 | Regression_eta: 0.0106 | Regression_pt: 0.2009 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3557 | Classification_binary: 3.6044 | Regression_cos_phi: 0.9764 | Regression_energy: 2.8382 | Regression_eta: 0.5187 | Regression_pt: 3.0224 | Regression_sin_phi: 0.7315 | LR: 9.98e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3020 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3045.5 | uss_MB: 2240.2 | child_rss_MB: 4123.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:24:19,017] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:58,938] INFO : Step 1700/50000 rank0 | Train Loss: 12.1787 | Train Loss Unweighted: 4.0310 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.5923 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1886 | Regression_eta: 0.0093 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4319 | Classification_binary: 3.5923 | Regression_cos_phi: 0.7128 | Regression_energy: 2.6085 | Regression_eta: 0.4579 | Regression_pt: 2.7934 | Regression_sin_phi: 0.5818 | LR: 9.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4373s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.3063 | max_optimizer_step: 0.1067 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3046.8 | uss_MB: 2242.6 | child_rss_MB: 4100.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:24:59,154] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:38,716] INFO : Step 1800/50000 rank0 | Train Loss: 12.8939 | Train Loss Unweighted: 4.0216 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.5469 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2033 | Regression_eta: 0.0146 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4193 | Classification_binary: 3.5469 | Regression_cos_phi: 0.7500 | Regression_energy: 2.8110 | Regression_eta: 0.7166 | Regression_pt: 3.0179 | Regression_sin_phi: 0.6322 | LR: 9.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4712s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2801 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3046.5 | uss_MB: 2242.8 | child_rss_MB: 4097.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:25:38,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:18,902] INFO : Step 1900/50000 rank0 | Train Loss: 12.5651 | Train Loss Unweighted: 4.0999 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.6587 | Regression_cos_phi: 0.0129 | Regression_energy: 0.1818 | Regression_eta: 0.0113 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4473 | Classification_binary: 3.6587 | Regression_cos_phi: 0.8597 | Regression_energy: 2.5144 | Regression_eta: 0.5560 | Regression_pt: 2.8364 | Regression_sin_phi: 0.6927 | LR: 9.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3103 | max_optimizer_step: 0.1063 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3046.5 | uss_MB: 2242.9 | child_rss_MB: 4094.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:26:19,116] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:58,534] INFO : Step 2000/50000 rank0 | Train Loss: 12.3343 | Train Loss Unweighted: 3.9901 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.5403 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1918 | Regression_eta: 0.0149 | Regression_pt: 0.1916 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3712 | Classification_binary: 3.5403 | Regression_cos_phi: 0.5970 | Regression_energy: 2.6519 | Regression_eta: 0.7317 | Regression_pt: 2.8821 | Regression_sin_phi: 0.5600 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3413s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1300 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3046.5 | uss_MB: 2243.1 | child_rss_MB: 4095.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:26:58,747] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:58,790] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:26:58,960] INFO : Running validation on rank0 +rank0 [2026-10-28 11:26:58,960] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:26:58,960] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:00,253] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:27:00,394] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:00,522] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:00,595] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:00,663] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:00,731] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:00,800] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:00,868] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:00,976] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:01,105] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:01,235] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:01,305] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:01,373] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:01,439] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:01,508] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:01,577] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:01,680] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:01,688] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:01,690] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:01,693] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:01,694] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:01,697] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:01,699] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:01,700] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:01,702] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:01,704] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:01,706] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:01,709] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:01,712] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:01,713] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:01,715] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:01,718] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:09,162] INFO : VALIDATION | Step=2000/50000 | Train Loss=12.3343 | Valid Loss=12.5441 | Train Loss Unweighted=3.9901 | Valid Loss Unweighted=4.0316 | Stale=0 | ETA=350.3m +rank0 [2026-10-28 11:27:09,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:09,234] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:09,237] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:09,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:11,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:11,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:12,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:13,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:13,279] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:13,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:13,340] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:13,342] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:13,486] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:15,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:15,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:16,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:17,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:17,401] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:17,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:17,459] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:17,462] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:27:17,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:19,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:20,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:20,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:21,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:21,486] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:21,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:21,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:22,639] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:24,086] INFO : Plotted jets +rank0 [2026-10-28 11:27:25,168] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:25,200] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:25,425] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:25,794] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:27,171] INFO : Plotted jets +rank0 [2026-10-28 11:27:28,276] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:28,308] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:28,509] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:28,916] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:30,293] INFO : Plotted jets +rank0 [2026-10-28 11:27:31,790] INFO : Plotted jet ratio +rank0 [2026-10-28 11:28:12,271] INFO : Step 2100/50000 rank0 | Train Loss: 12.3404 | Train Loss Unweighted: 3.9792 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.5360 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1882 | Regression_eta: 0.0127 | Regression_pt: 0.1862 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4525 | Classification_binary: 3.5360 | Regression_cos_phi: 0.6734 | Regression_energy: 2.6027 | Regression_eta: 0.6237 | Regression_pt: 2.8006 | Regression_sin_phi: 0.6515 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4513s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1775 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3250.6 | uss_MB: 2450.6 | child_rss_MB: 4046.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:28:12,483] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:52,054] INFO : Step 2200/50000 rank0 | Train Loss: 11.9485 | Train Loss Unweighted: 3.8666 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.4376 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1811 | Regression_eta: 0.0139 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2781 | Classification_binary: 3.4376 | Regression_cos_phi: 0.7587 | Regression_energy: 2.5041 | Regression_eta: 0.6822 | Regression_pt: 2.7482 | Regression_sin_phi: 0.5396 | LR: 9.96e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6120s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0774 | max_forward: 0.1050 | max_loss: 0.0031 | max_backward: 0.3954 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3252.5 | uss_MB: 2452.7 | child_rss_MB: 4043.5 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:28:52,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:31,811] INFO : Step 2300/50000 rank0 | Train Loss: 12.1868 | Train Loss Unweighted: 3.9852 | Train Loss Components: Classification: 0.0319 | Classification_binary: 3.5422 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1910 | Regression_eta: 0.0104 | Regression_pt: 0.1882 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3374 | Classification_binary: 3.5422 | Regression_cos_phi: 0.6791 | Regression_energy: 2.6414 | Regression_eta: 0.5127 | Regression_pt: 2.8310 | Regression_sin_phi: 0.6431 | LR: 9.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3060 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3929 | max_tensorboard_flush: 0.0147 | Memory: rss_MB: 3250.1 | uss_MB: 2450.7 | child_rss_MB: 4046.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:29:32,026] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:11,997] INFO : Step 2400/50000 rank0 | Train Loss: 11.9228 | Train Loss Unweighted: 3.8706 | Train Loss Components: Classification: 0.0310 | Classification_binary: 3.4490 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1796 | Regression_eta: 0.0144 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2989 | Classification_binary: 3.4490 | Regression_cos_phi: 0.6911 | Regression_energy: 2.4830 | Regression_eta: 0.7052 | Regression_pt: 2.6284 | Regression_sin_phi: 0.6672 | LR: 9.95e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3108 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3250.1 | uss_MB: 2450.7 | child_rss_MB: 4041.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:30:12,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:51,654] INFO : Step 2500/50000 rank0 | Train Loss: 11.5616 | Train Loss Unweighted: 3.9130 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.5057 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1728 | Regression_eta: 0.0102 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3692 | Classification_binary: 3.5057 | Regression_cos_phi: 0.6821 | Regression_energy: 2.3895 | Regression_eta: 0.4997 | Regression_pt: 2.5917 | Regression_sin_phi: 0.5238 | LR: 9.94e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1318 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3250.0 | uss_MB: 2450.7 | child_rss_MB: 4038.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:30:51,867] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:31,571] INFO : Step 2600/50000 rank0 | Train Loss: 12.4914 | Train Loss Unweighted: 3.9695 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.5127 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1947 | Regression_eta: 0.0131 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1800 | Classification_binary: 3.5127 | Regression_cos_phi: 0.8187 | Regression_energy: 2.6919 | Regression_eta: 0.6414 | Regression_pt: 2.9581 | Regression_sin_phi: 0.6887 | LR: 9.94e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.5475s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3912 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3116.1 | uss_MB: 2316.8 | child_rss_MB: 4041.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:31:31,784] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:11,991] INFO : Step 2700/50000 rank0 | Train Loss: 12.6956 | Train Loss Unweighted: 3.9876 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.5223 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1977 | Regression_eta: 0.0128 | Regression_pt: 0.1993 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3184 | Classification_binary: 3.5223 | Regression_cos_phi: 0.8541 | Regression_energy: 2.7339 | Regression_eta: 0.6301 | Regression_pt: 2.9976 | Regression_sin_phi: 0.6390 | LR: 9.94e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3991s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1406 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.2870 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3117.1 | uss_MB: 2318.9 | child_rss_MB: 4038.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:32:12,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:52,097] INFO : Step 2800/50000 rank0 | Train Loss: 11.9806 | Train Loss Unweighted: 3.8973 | Train Loss Components: Classification: 0.0313 | Classification_binary: 3.4799 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1748 | Regression_eta: 0.0169 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3093 | Classification_binary: 3.4799 | Regression_cos_phi: 0.7144 | Regression_energy: 2.4173 | Regression_eta: 0.8299 | Regression_pt: 2.5985 | Regression_sin_phi: 0.6313 | LR: 9.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3117.1 | uss_MB: 2319.0 | child_rss_MB: 4032.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:32:52,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:31,597] INFO : Step 2900/50000 rank0 | Train Loss: 11.6618 | Train Loss Unweighted: 3.7539 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.3450 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1726 | Regression_eta: 0.0156 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3257 | Classification_binary: 3.3450 | Regression_cos_phi: 0.6325 | Regression_energy: 2.3864 | Regression_eta: 0.7666 | Regression_pt: 2.5209 | Regression_sin_phi: 0.6846 | LR: 9.93e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1843 | max_loss: 0.0029 | max_backward: 0.2026 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3117.1 | uss_MB: 2319.1 | child_rss_MB: 4034.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:33:31,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:11,616] INFO : Step 3000/50000 rank0 | Train Loss: 11.7576 | Train Loss Unweighted: 3.9013 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.4849 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1769 | Regression_eta: 0.0121 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3248 | Classification_binary: 3.4849 | Regression_cos_phi: 0.7113 | Regression_energy: 2.4456 | Regression_eta: 0.5950 | Regression_pt: 2.6380 | Regression_sin_phi: 0.5581 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0030 | max_backward: 0.3094 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3117.1 | uss_MB: 2319.1 | child_rss_MB: 4034.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:34:11,828] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:11,846] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:34:12,119] INFO : Running validation on rank0 +rank0 [2026-10-28 11:34:12,119] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:34:12,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:34:13,298] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:34:13,441] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:34:13,570] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:34:13,644] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:34:13,711] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:34:13,778] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:34:13,846] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:34:13,913] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:34:14,019] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:34:14,148] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:34:14,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:34:14,348] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:34:14,415] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:34:14,481] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:34:14,549] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:34:14,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:34:14,718] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:34:14,725] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:34:14,728] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:34:14,731] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:34:14,732] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:34:14,735] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:34:14,737] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:34:14,740] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:34:14,741] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:34:14,848] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:34:14,851] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:34:14,852] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:34:14,855] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:34:14,858] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:34:14,861] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:34:14,863] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:22,272] INFO : VALIDATION | Step=3000/50000 | Train Loss=11.7576 | Valid Loss=11.7898 | Train Loss Unweighted=3.9013 | Valid Loss Unweighted=3.8000 | Stale=0 | ETA=341.7m +rank0 [2026-10-28 11:34:22,282] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:22,348] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:22,351] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:22,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:24,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:24,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:25,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:26,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:26,121] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:26,124] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:26,180] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:26,183] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:26,257] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:28,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:28,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:29,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:30,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:30,116] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:30,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:30,173] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:34:30,176] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:34:30,320] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:32,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:32,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:33,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:34,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:34,209] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:34,216] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:34,470] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:34,914] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:36,751] INFO : Plotted jets +rank0 [2026-10-28 11:34:37,826] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:37,857] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:38,081] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:38,448] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:39,813] INFO : Plotted jets +rank0 [2026-10-28 11:34:40,920] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:40,953] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:41,153] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:41,561] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:42,933] INFO : Plotted jets +rank0 [2026-10-28 11:34:44,405] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:24,177] INFO : Step 3100/50000 rank0 | Train Loss: 12.0916 | Train Loss Unweighted: 3.8176 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1935 | Regression_eta: 0.0103 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2855 | Classification_binary: 3.3700 | Regression_cos_phi: 0.7046 | Regression_energy: 2.6759 | Regression_eta: 0.5051 | Regression_pt: 2.8672 | Regression_sin_phi: 0.6833 | LR: 9.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4475s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1718 | max_loss: 0.0030 | max_backward: 0.2025 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.2892 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3334.8 | uss_MB: 2538.7 | child_rss_MB: 4033.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:35:24,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:03,664] INFO : Step 3200/50000 rank0 | Train Loss: 11.2594 | Train Loss Unweighted: 3.8042 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.4140 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1631 | Regression_eta: 0.0132 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2337 | Classification_binary: 3.4140 | Regression_cos_phi: 0.6444 | Regression_energy: 2.2557 | Regression_eta: 0.6495 | Regression_pt: 2.4733 | Regression_sin_phi: 0.5887 | LR: 9.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4461s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1834 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4034.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:36:03,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:42,471] INFO : Step 3300/50000 rank0 | Train Loss: 11.7776 | Train Loss Unweighted: 3.7029 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.2734 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1820 | Regression_eta: 0.0132 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2512 | Classification_binary: 3.2734 | Regression_cos_phi: 0.6102 | Regression_energy: 2.5172 | Regression_eta: 0.6465 | Regression_pt: 2.7445 | Regression_sin_phi: 0.7346 | LR: 9.90e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1134 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4032.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:36:42,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:22,527] INFO : Step 3400/50000 rank0 | Train Loss: 11.2672 | Train Loss Unweighted: 3.6481 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.2413 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1719 | Regression_eta: 0.0111 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2037 | Classification_binary: 3.2413 | Regression_cos_phi: 0.6797 | Regression_energy: 2.3771 | Regression_eta: 0.5458 | Regression_pt: 2.6235 | Regression_sin_phi: 0.5961 | LR: 9.90e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4825s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.2600 | max_optimizer_step: 0.1103 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4031.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:37:22,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:02,752] INFO : Step 3500/50000 rank0 | Train Loss: 11.5238 | Train Loss Unweighted: 3.7867 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.3813 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1689 | Regression_eta: 0.0140 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2443 | Classification_binary: 3.3813 | Regression_cos_phi: 0.6963 | Regression_energy: 2.3354 | Regression_eta: 0.6866 | Regression_pt: 2.5868 | Regression_sin_phi: 0.5931 | LR: 9.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6858s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0033 | max_backward: 0.3927 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3335.5 | uss_MB: 2540.9 | child_rss_MB: 4035.9 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:38:02,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:42,253] INFO : Step 3600/50000 rank0 | Train Loss: 10.5003 | Train Loss Unweighted: 3.5529 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.1695 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1663 | Regression_eta: 0.0089 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1620 | Classification_binary: 3.1695 | Regression_cos_phi: 0.5276 | Regression_energy: 2.3000 | Regression_eta: 0.4385 | Regression_pt: 2.4854 | Regression_sin_phi: 0.4173 | LR: 9.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2935 | max_optimizer_step: 0.1090 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3186.6 | uss_MB: 2390.1 | child_rss_MB: 4030.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:42,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:22,337] INFO : Step 3700/50000 rank0 | Train Loss: 10.5089 | Train Loss Unweighted: 3.6312 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.2631 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1564 | Regression_eta: 0.0101 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1986 | Classification_binary: 3.2631 | Regression_cos_phi: 0.5470 | Regression_energy: 2.1623 | Regression_eta: 0.4981 | Regression_pt: 2.3509 | Regression_sin_phi: 0.4889 | LR: 9.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5012s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3038 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3642 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3187.6 | uss_MB: 2391.6 | child_rss_MB: 4029.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:39:22,548] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:02,675] INFO : Step 3800/50000 rank0 | Train Loss: 10.7211 | Train Loss Unweighted: 3.6064 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.2434 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1502 | Regression_eta: 0.0133 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1555 | Classification_binary: 3.2434 | Regression_cos_phi: 0.5974 | Regression_energy: 2.0770 | Regression_eta: 0.6556 | Regression_pt: 2.2581 | Regression_sin_phi: 0.7340 | LR: 9.87e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4508s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0508 | max_backward: 0.2031 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 3187.6 | uss_MB: 2391.7 | child_rss_MB: 4027.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:02,889] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:42,444] INFO : Step 3900/50000 rank0 | Train Loss: 12.7464 | Train Loss Unweighted: 3.6417 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.1699 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1969 | Regression_eta: 0.0300 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2497 | Classification_binary: 3.1699 | Regression_cos_phi: 0.6527 | Regression_energy: 2.7235 | Regression_eta: 1.4724 | Regression_pt: 2.9484 | Regression_sin_phi: 0.5297 | LR: 9.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5703s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1862 | max_loss: 0.0030 | max_backward: 0.2869 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3187.8 | uss_MB: 2394.3 | child_rss_MB: 4029.2 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:40:42,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:21,968] INFO : Step 4000/50000 rank0 | Train Loss: 11.5654 | Train Loss Unweighted: 3.6366 | Train Loss Components: Classification: 0.0310 | Classification_binary: 3.2277 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1725 | Regression_eta: 0.0145 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2979 | Classification_binary: 3.2277 | Regression_cos_phi: 0.7732 | Regression_energy: 2.3861 | Regression_eta: 0.7121 | Regression_pt: 2.5305 | Regression_sin_phi: 0.6380 | LR: 9.86e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4429s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0331 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3187.6 | uss_MB: 2392.3 | child_rss_MB: 4025.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:41:22,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:22,214] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:41:22,386] INFO : Running validation on rank0 +rank0 [2026-10-28 11:41:22,386] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:41:22,386] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:41:23,700] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:41:23,843] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:41:23,972] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:41:24,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:41:24,111] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:41:24,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:41:24,247] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:41:24,316] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:41:24,425] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:41:24,554] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:41:24,682] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:41:24,752] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:41:24,819] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:41:24,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:41:24,956] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:41:25,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:41:25,136] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:41:25,142] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:41:25,145] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:41:25,148] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:41:25,149] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:41:25,152] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:41:25,154] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:41:25,155] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:41:25,157] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:41:25,160] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:41:25,161] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:41:25,163] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:41:25,166] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:41:25,167] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:41:25,169] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:41:25,172] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:41:32,615] INFO : VALIDATION | Step=4000/50000 | Train Loss=11.5654 | Valid Loss=11.1201 | Train Loss Unweighted=3.6366 | Valid Loss Unweighted=3.6235 | Stale=0 | ETA=333.3m +rank0 [2026-10-28 11:41:32,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:32,693] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:41:32,696] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:32,842] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:34,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:35,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:35,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:36,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:36,741] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:41:36,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:36,799] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:41:36,802] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:36,878] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:38,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:39,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:40,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:40,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:40,815] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:41:40,819] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:40,873] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:41:40,876] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:41:41,023] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:42,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:43,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:44,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:44,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:44,916] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:41:44,923] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:45,176] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:45,624] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:47,461] INFO : Plotted jets +rank0 [2026-10-28 11:41:48,550] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:48,582] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:48,803] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:49,177] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:50,550] INFO : Plotted jets +rank0 [2026-10-28 11:41:51,661] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:51,693] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:41:51,894] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:41:52,309] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:53,691] INFO : Plotted jets +rank0 [2026-10-28 11:41:55,161] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:34,831] INFO : Step 4100/50000 rank0 | Train Loss: 11.0707 | Train Loss Unweighted: 3.6203 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.2302 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1647 | Regression_eta: 0.0122 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2511 | Classification_binary: 3.2302 | Regression_cos_phi: 0.6834 | Regression_energy: 2.2772 | Regression_eta: 0.6017 | Regression_pt: 2.4532 | Regression_sin_phi: 0.5739 | LR: 9.85e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1280 | max_loss: 0.0034 | max_backward: 0.2028 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4020.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:42:35,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:14,956] INFO : Step 4200/50000 rank0 | Train Loss: 10.5140 | Train Loss Unweighted: 3.5173 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.1305 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1676 | Regression_eta: 0.0093 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1625 | Classification_binary: 3.1305 | Regression_cos_phi: 0.5208 | Regression_energy: 2.3172 | Regression_eta: 0.4581 | Regression_pt: 2.5147 | Regression_sin_phi: 0.4102 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5795s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3060 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4021.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:43:15,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:55,324] INFO : Step 4300/50000 rank0 | Train Loss: 10.4742 | Train Loss Unweighted: 3.4542 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.0921 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1526 | Regression_eta: 0.0118 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1382 | Classification_binary: 3.0921 | Regression_cos_phi: 0.6938 | Regression_energy: 2.1098 | Regression_eta: 0.5806 | Regression_pt: 2.2467 | Regression_sin_phi: 0.6129 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0824 | max_backward: 0.2032 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:43:55,538] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:35,056] INFO : Step 4400/50000 rank0 | Train Loss: 10.1726 | Train Loss Unweighted: 3.4727 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.1148 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1536 | Regression_eta: 0.0117 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0674 | Classification_binary: 3.1148 | Regression_cos_phi: 0.5871 | Regression_energy: 2.1243 | Regression_eta: 0.5743 | Regression_pt: 2.2673 | Regression_sin_phi: 0.4375 | LR: 9.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5413s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1001 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.2031 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:44:35,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:14,872] INFO : Step 4500/50000 rank0 | Train Loss: 11.7776 | Train Loss Unweighted: 3.6103 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.1637 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1918 | Regression_eta: 0.0155 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2614 | Classification_binary: 3.1637 | Regression_cos_phi: 0.5703 | Regression_energy: 2.6517 | Regression_eta: 0.7615 | Regression_pt: 2.8945 | Regression_sin_phi: 0.4745 | LR: 9.82e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5795s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3062 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4049.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:45:15,083] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:55,956] INFO : Step 4600/50000 rank0 | Train Loss: 11.5346 | Train Loss Unweighted: 3.5693 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.1461 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1791 | Regression_eta: 0.0131 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2191 | Classification_binary: 3.1461 | Regression_cos_phi: 0.6960 | Regression_energy: 2.4769 | Regression_eta: 0.6457 | Regression_pt: 2.7108 | Regression_sin_phi: 0.6400 | LR: 9.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1048 | max_loss: 0.0033 | max_backward: 0.3195 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3421.7 | uss_MB: 2632.1 | child_rss_MB: 4016.0 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:45:56,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:36,275] INFO : Step 4700/50000 rank0 | Train Loss: 10.5234 | Train Loss Unweighted: 3.5185 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1449 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1607 | Regression_eta: 0.0106 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1388 | Classification_binary: 3.1449 | Regression_cos_phi: 0.5906 | Regression_energy: 2.2222 | Regression_eta: 0.5190 | Regression_pt: 2.3577 | Regression_sin_phi: 0.5504 | LR: 9.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4174s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1067 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1095 | max_tensorboard_logging: 0.3057 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3258.1 | uss_MB: 2467.6 | child_rss_MB: 4013.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:46:36,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:16,258] INFO : Step 4800/50000 rank0 | Train Loss: 10.3396 | Train Loss Unweighted: 3.4477 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1525 | Regression_eta: 0.0111 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1254 | Classification_binary: 3.0859 | Regression_cos_phi: 0.6664 | Regression_energy: 2.1090 | Regression_eta: 0.5432 | Regression_pt: 2.2920 | Regression_sin_phi: 0.5178 | LR: 9.80e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5667s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.3080 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 3258.1 | uss_MB: 2467.6 | child_rss_MB: 4015.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:47:16,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:55,232] INFO : Step 4900/50000 rank0 | Train Loss: 10.0105 | Train Loss Unweighted: 3.3531 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.0015 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1492 | Regression_eta: 0.0101 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0722 | Classification_binary: 3.0015 | Regression_cos_phi: 0.6114 | Regression_energy: 2.0634 | Regression_eta: 0.4969 | Regression_pt: 2.2287 | Regression_sin_phi: 0.5364 | LR: 9.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1799 | max_loss: 0.0030 | max_backward: 0.2027 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3258.1 | uss_MB: 2468.1 | child_rss_MB: 4016.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:47:55,453] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:36,174] INFO : Step 5000/50000 rank0 | Train Loss: 10.7358 | Train Loss Unweighted: 3.5550 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.1712 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1633 | Regression_eta: 0.0108 | Regression_pt: 0.1641 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0774 | Classification_binary: 3.1712 | Regression_cos_phi: 0.6698 | Regression_energy: 2.2584 | Regression_eta: 0.5286 | Regression_pt: 2.4695 | Regression_sin_phi: 0.5610 | LR: 9.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3998s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1310 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1079 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3258.1 | uss_MB: 2468.1 | child_rss_MB: 4022.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:48:36,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:36,408] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:48:36,581] INFO : Running validation on rank0 +rank0 [2026-10-28 11:48:36,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:48:36,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:48:37,864] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:48:38,008] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:48:38,136] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:48:38,208] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:48:38,275] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:48:38,342] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:48:38,411] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:48:38,479] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:48:38,590] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:48:38,719] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:48:38,847] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:48:38,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:48:38,984] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:48:39,050] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:48:39,118] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:48:39,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:48:39,296] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:48:39,304] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:48:39,307] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:48:39,310] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:48:39,311] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:48:39,314] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:48:39,317] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:48:39,320] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:48:39,321] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:48:39,323] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:48:39,326] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:48:39,326] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:48:39,330] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:48:39,332] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:48:39,335] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:48:39,336] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:48:46,637] INFO : VALIDATION | Step=5000/50000 | Train Loss=10.7358 | Valid Loss=10.7371 | Train Loss Unweighted=3.5550 | Valid Loss Unweighted=3.4633 | Stale=0 | ETA=326.0m +rank0 [2026-10-28 11:48:46,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:46,710] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:48:46,714] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:46,779] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:48,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:49,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:50,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:50,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:50,870] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:48:50,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:50,936] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:48:50,938] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:48:51,090] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:52,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:53,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:54,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:54,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:55,019] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:48:55,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:55,083] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:48:55,086] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:48:55,237] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:57,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:57,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:58,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:59,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:59,145] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:48:59,153] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:59,412] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:59,869] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:01,747] INFO : Plotted jets +rank0 [2026-10-28 11:49:02,840] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:02,875] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:03,103] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:03,476] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:04,848] INFO : Plotted jets +rank0 [2026-10-28 11:49:05,968] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:06,002] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:06,204] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:06,621] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:08,001] INFO : Plotted jets +rank0 [2026-10-28 11:49:09,111] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:49,518] INFO : Step 5100/50000 rank0 | Train Loss: 10.5352 | Train Loss Unweighted: 3.4065 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.0227 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1655 | Regression_eta: 0.0102 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0138 | Classification_binary: 3.0227 | Regression_cos_phi: 0.6897 | Regression_energy: 2.2890 | Regression_eta: 0.5021 | Regression_pt: 2.4650 | Regression_sin_phi: 0.5529 | LR: 9.77e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2651 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.3949 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4019.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:49:49,730] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:29,472] INFO : Step 5200/50000 rank0 | Train Loss: 10.6893 | Train Loss Unweighted: 3.4241 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.0325 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1678 | Regression_eta: 0.0111 | Regression_pt: 0.1678 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0718 | Classification_binary: 3.0325 | Regression_cos_phi: 0.6188 | Regression_energy: 2.3204 | Regression_eta: 0.5440 | Regression_pt: 2.5251 | Regression_sin_phi: 0.5767 | LR: 9.76e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.4695s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3094 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4041.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:50:29,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:08,793] INFO : Step 5300/50000 rank0 | Train Loss: 9.7687 | Train Loss Unweighted: 3.4334 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.0874 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1493 | Regression_eta: 0.0093 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9793 | Classification_binary: 3.0874 | Regression_cos_phi: 0.4698 | Regression_energy: 2.0643 | Regression_eta: 0.4546 | Regression_pt: 2.2381 | Regression_sin_phi: 0.4753 | LR: 9.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5943s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1036 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2027 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.3393 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4015.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:51:09,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:48,275] INFO : Step 5400/50000 rank0 | Train Loss: 10.4975 | Train Loss Unweighted: 3.4566 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.0878 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1565 | Regression_eta: 0.0109 | Regression_pt: 0.1541 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1208 | Classification_binary: 3.0878 | Regression_cos_phi: 0.6981 | Regression_energy: 2.1643 | Regression_eta: 0.5348 | Regression_pt: 2.3183 | Regression_sin_phi: 0.5734 | LR: 9.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5148s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0235 | max_forward: 0.0780 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.3155 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:51:48,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:28,801] INFO : Step 5500/50000 rank0 | Train Loss: 10.6236 | Train Loss Unweighted: 3.2562 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.8804 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1579 | Regression_eta: 0.0176 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1114 | Classification_binary: 2.8804 | Regression_cos_phi: 0.7031 | Regression_energy: 2.1833 | Regression_eta: 0.8632 | Regression_pt: 2.3055 | Regression_sin_phi: 0.5768 | LR: 9.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5820s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3033 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3966 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3483.1 | uss_MB: 2693.6 | child_rss_MB: 4017.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:29,018] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:07,705] INFO : Step 5600/50000 rank0 | Train Loss: 10.1159 | Train Loss Unweighted: 3.3475 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.9753 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1630 | Regression_eta: 0.0116 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9548 | Classification_binary: 2.9753 | Regression_cos_phi: 0.5776 | Regression_energy: 2.2540 | Regression_eta: 0.5712 | Regression_pt: 2.3983 | Regression_sin_phi: 0.3848 | LR: 9.72e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5410s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0445 | max_backward: 0.2026 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3483.1 | uss_MB: 2693.6 | child_rss_MB: 4012.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:53:07,918] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:48,458] INFO : Step 5700/50000 rank0 | Train Loss: 10.1841 | Train Loss Unweighted: 3.4032 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.0420 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1563 | Regression_eta: 0.0122 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9941 | Classification_binary: 3.0420 | Regression_cos_phi: 0.6416 | Regression_energy: 2.1615 | Regression_eta: 0.6015 | Regression_pt: 2.2733 | Regression_sin_phi: 0.4701 | LR: 9.71e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3095 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3331.3 | uss_MB: 2541.7 | child_rss_MB: 4012.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:53:48,673] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:28,582] INFO : Step 5800/50000 rank0 | Train Loss: 10.2215 | Train Loss Unweighted: 3.3291 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9586 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1607 | Regression_eta: 0.0096 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9765 | Classification_binary: 2.9586 | Regression_cos_phi: 0.6407 | Regression_energy: 2.2223 | Regression_eta: 0.4706 | Regression_pt: 2.3598 | Regression_sin_phi: 0.5930 | LR: 9.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5301s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.3093 | max_optimizer_step: 0.1099 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3331.3 | uss_MB: 2541.7 | child_rss_MB: 4012.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:28,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:08,834] INFO : Step 5900/50000 rank0 | Train Loss: 10.3740 | Train Loss Unweighted: 3.2798 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.9110 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1561 | Regression_eta: 0.0118 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1644 | Classification_binary: 2.9110 | Regression_cos_phi: 0.6861 | Regression_energy: 2.1586 | Regression_eta: 0.5816 | Regression_pt: 2.2988 | Regression_sin_phi: 0.5735 | LR: 9.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2825 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3815 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3331.3 | uss_MB: 2541.9 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:55:09,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:48,775] INFO : Step 6000/50000 rank0 | Train Loss: 10.4734 | Train Loss Unweighted: 3.3621 | Train Loss Components: Classification: 0.0262 | Classification_binary: 2.9774 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1660 | Regression_eta: 0.0087 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0966 | Classification_binary: 2.9774 | Regression_cos_phi: 0.6844 | Regression_energy: 2.2952 | Regression_eta: 0.4285 | Regression_pt: 2.4748 | Regression_sin_phi: 0.5164 | LR: 9.68e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4426s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.3070 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3331.3 | uss_MB: 2542.0 | child_rss_MB: 4033.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:55:48,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:49,006] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:55:49,179] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:49,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:49,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:50,420] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:55:50,564] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:50,693] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:50,765] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:50,832] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:50,900] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:50,969] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:51,036] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:51,146] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:51,274] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:51,403] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:51,473] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:51,541] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:51,609] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:51,677] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:51,745] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:51,845] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:51,852] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:51,854] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:51,857] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:51,858] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:51,861] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:51,864] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:51,864] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:51,867] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:51,870] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:51,870] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:51,873] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:51,876] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:51,876] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:51,879] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:51,880] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:59,263] INFO : VALIDATION | Step=6000/50000 | Train Loss=10.4734 | Valid Loss=10.2158 | Train Loss Unweighted=3.3621 | Valid Loss Unweighted=3.3282 | Stale=0 | ETA=318.5m +rank0 [2026-10-28 11:55:59,273] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:59,340] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:59,344] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:59,489] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:01,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:01,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:02,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:03,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:03,471] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:56:03,474] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:56:03,539] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:56:03,542] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:03,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:05,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:06,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:06,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:07,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:07,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:56:07,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:56:07,543] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:56:07,546] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:56:07,696] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:09,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:10,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:10,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:11,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:11,607] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:56:11,615] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:56:11,869] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:56:12,318] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:14,218] INFO : Plotted jets +rank0 [2026-10-28 11:56:15,307] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:15,341] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:15,565] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:15,937] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:17,313] INFO : Plotted jets +rank0 [2026-10-28 11:56:18,431] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:18,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:56:18,735] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:56:19,148] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:20,532] INFO : Plotted jets +rank0 [2026-10-28 11:56:21,625] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:01,411] INFO : Step 6100/50000 rank0 | Train Loss: 10.0922 | Train Loss Unweighted: 3.3107 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.9494 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1536 | Regression_eta: 0.0093 | Regression_pt: 0.1539 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0638 | Classification_binary: 2.9494 | Regression_cos_phi: 0.6051 | Regression_energy: 2.1238 | Regression_eta: 0.4584 | Regression_pt: 2.3150 | Regression_sin_phi: 0.5767 | LR: 9.67e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0544 | max_backward: 0.2029 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3547.6 | uss_MB: 2761.4 | child_rss_MB: 4010.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:57:01,625] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:41,438] INFO : Step 6200/50000 rank0 | Train Loss: 9.9202 | Train Loss Unweighted: 3.2990 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.9514 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1459 | Regression_eta: 0.0109 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0145 | Classification_binary: 2.9514 | Regression_cos_phi: 0.6173 | Regression_energy: 2.0178 | Regression_eta: 0.5345 | Regression_pt: 2.2171 | Regression_sin_phi: 0.5676 | LR: 9.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5720s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3083 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:57:41,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:21,192] INFO : Step 6300/50000 rank0 | Train Loss: 9.5818 | Train Loss Unweighted: 3.3361 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9944 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1467 | Regression_eta: 0.0082 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.9944 | Regression_cos_phi: 0.5355 | Regression_energy: 2.0288 | Regression_eta: 0.4053 | Regression_pt: 2.2366 | Regression_sin_phi: 0.4650 | LR: 9.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5246s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0741 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2032 | max_optimizer_step: 0.1820 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4004.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:58:21,419] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:00,640] INFO : Step 6400/50000 rank0 | Train Loss: 10.5963 | Train Loss Unweighted: 3.3863 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.9966 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1681 | Regression_eta: 0.0120 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0558 | Classification_binary: 2.9966 | Regression_cos_phi: 0.6322 | Regression_energy: 2.3243 | Regression_eta: 0.5901 | Regression_pt: 2.5013 | Regression_sin_phi: 0.4961 | LR: 9.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4715s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0537 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:59:00,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:40,499] INFO : Step 6500/50000 rank0 | Train Loss: 10.2834 | Train Loss Unweighted: 3.2737 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8982 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1617 | Regression_eta: 0.0121 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9700 | Classification_binary: 2.8982 | Regression_cos_phi: 0.6373 | Regression_energy: 2.2364 | Regression_eta: 0.5952 | Regression_pt: 2.3953 | Regression_sin_phi: 0.5509 | LR: 9.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5580s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0921 | max_backward: 0.2027 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4003.3 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:59:40,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:21,097] INFO : Step 6600/50000 rank0 | Train Loss: 10.0484 | Train Loss Unweighted: 3.2922 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9384 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1493 | Regression_eta: 0.0108 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9873 | Classification_binary: 2.9384 | Regression_cos_phi: 0.6692 | Regression_energy: 2.0646 | Regression_eta: 0.5322 | Regression_pt: 2.2483 | Regression_sin_phi: 0.6084 | LR: 9.62e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3084 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4031.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:00:21,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:00,933] INFO : Step 6700/50000 rank0 | Train Loss: 10.1948 | Train Loss Unweighted: 3.3072 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.9468 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1526 | Regression_eta: 0.0124 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9672 | Classification_binary: 2.9468 | Regression_cos_phi: 0.7079 | Regression_energy: 2.1107 | Regression_eta: 0.6072 | Regression_pt: 2.2825 | Regression_sin_phi: 0.5724 | LR: 9.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2269 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3402.6 | uss_MB: 2615.4 | child_rss_MB: 4007.8 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:01:01,148] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:40,577] INFO : Step 6800/50000 rank0 | Train Loss: 9.6889 | Train Loss Unweighted: 3.2773 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.9353 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1459 | Regression_eta: 0.0107 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9472 | Classification_binary: 2.9353 | Regression_cos_phi: 0.6053 | Regression_energy: 2.0170 | Regression_eta: 0.5248 | Regression_pt: 2.1903 | Regression_sin_phi: 0.4690 | LR: 9.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3508s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1028 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1079 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3402.6 | uss_MB: 2615.5 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:01:40,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:19,961] INFO : Step 6900/50000 rank0 | Train Loss: 9.3526 | Train Loss Unweighted: 3.1332 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.7952 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1465 | Regression_eta: 0.0078 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9650 | Classification_binary: 2.7952 | Regression_cos_phi: 0.5339 | Regression_energy: 2.0264 | Regression_eta: 0.3810 | Regression_pt: 2.1708 | Regression_sin_phi: 0.4804 | LR: 9.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0567 | max_backward: 0.2025 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3816 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3402.6 | uss_MB: 2615.6 | child_rss_MB: 4006.4 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:02:20,173] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:00,160] INFO : Step 7000/50000 rank0 | Train Loss: 10.1169 | Train Loss Unweighted: 3.2794 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9182 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1545 | Regression_eta: 0.0110 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9900 | Classification_binary: 2.9182 | Regression_cos_phi: 0.6860 | Regression_energy: 2.1371 | Regression_eta: 0.5420 | Regression_pt: 2.2857 | Regression_sin_phi: 0.5580 | LR: 9.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4417s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1820 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1095 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3402.6 | uss_MB: 2615.7 | child_rss_MB: 4007.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:03:00,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:00,410] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 12:03:00,589] INFO : Running validation on rank0 +rank0 [2026-10-28 12:03:00,589] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:03:00,589] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:03:01,892] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 12:03:02,034] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:03:02,164] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:03:02,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:03:02,303] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:03:02,370] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:03:02,439] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:03:02,506] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:03:02,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:03:02,746] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:03:02,875] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:03:02,950] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:03:03,018] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:03:03,085] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:03:03,154] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:03:03,222] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:03:03,323] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:03:03,329] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:03:03,332] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:03:03,333] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:03:03,336] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:03:03,337] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:03:03,339] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:03:03,342] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:03:03,343] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:03:03,345] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:03:03,348] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:03:03,349] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:03:03,351] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:03:03,354] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:03:03,355] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:03:03,357] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:03:10,814] INFO : VALIDATION | Step=7000/50000 | Train Loss=10.1169 | Valid Loss=9.9681 | Train Loss Unweighted=3.2794 | Valid Loss Unweighted=3.2478 | Stale=0 | ETA=311.0m +rank0 [2026-10-28 12:03:10,824] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:10,884] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:03:10,887] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:11,034] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:12,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:13,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:14,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:15,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:15,149] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:03:15,152] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:15,210] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:03:15,212] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:15,293] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:17,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:17,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:18,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:19,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:19,223] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:03:19,226] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:19,291] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:03:19,294] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:03:19,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:21,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:21,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:22,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:23,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:23,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:03:23,280] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:23,534] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:23,980] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:25,834] INFO : Plotted jets +rank0 [2026-10-28 12:03:26,918] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:26,950] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:27,174] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:27,544] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:28,906] INFO : Plotted jets +rank0 [2026-10-28 12:03:30,013] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:30,045] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 12:03:30,241] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 12:03:30,650] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:32,044] INFO : Plotted jets +rank0 [2026-10-28 12:03:33,131] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:12,724] INFO : Step 7100/50000 rank0 | Train Loss: 10.0366 | Train Loss Unweighted: 3.2548 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8943 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1535 | Regression_eta: 0.0096 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0569 | Classification_binary: 2.8943 | Regression_cos_phi: 0.6837 | Regression_energy: 2.1222 | Regression_eta: 0.4707 | Regression_pt: 2.3034 | Regression_sin_phi: 0.5054 | LR: 9.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5407s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2787 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4008.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:12,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:52,271] INFO : Step 7200/50000 rank0 | Train Loss: 9.8051 | Train Loss Unweighted: 3.3297 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9756 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1530 | Regression_eta: 0.0093 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9114 | Classification_binary: 2.9756 | Regression_cos_phi: 0.4991 | Regression_energy: 2.1155 | Regression_eta: 0.4568 | Regression_pt: 2.3030 | Regression_sin_phi: 0.5437 | LR: 9.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1663 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.2909 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:52,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:31,503] INFO : Step 7300/50000 rank0 | Train Loss: 10.0799 | Train Loss Unweighted: 3.3576 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.9919 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1575 | Regression_eta: 0.0084 | Regression_pt: 0.1571 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0559 | Classification_binary: 2.9919 | Regression_cos_phi: 0.5677 | Regression_energy: 2.1781 | Regression_eta: 0.4134 | Regression_pt: 2.3638 | Regression_sin_phi: 0.5091 | LR: 9.53e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4971s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2245 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:05:31,715] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:10,955] INFO : Step 7400/50000 rank0 | Train Loss: 10.3337 | Train Loss Unweighted: 3.2562 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8856 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1589 | Regression_eta: 0.0106 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9747 | Classification_binary: 2.8856 | Regression_cos_phi: 0.7083 | Regression_energy: 2.1969 | Regression_eta: 0.5223 | Regression_pt: 2.3264 | Regression_sin_phi: 0.7196 | LR: 9.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5363s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2692 | max_optimizer_step: 0.1146 | max_tensorboard_logging: 0.3852 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4009.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:11,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:51,091] INFO : Step 7500/50000 rank0 | Train Loss: 10.1096 | Train Loss Unweighted: 3.2043 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8364 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1604 | Regression_eta: 0.0110 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9760 | Classification_binary: 2.8364 | Regression_cos_phi: 0.6698 | Regression_energy: 2.2183 | Regression_eta: 0.5417 | Regression_pt: 2.3062 | Regression_sin_phi: 0.5612 | LR: 9.51e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5191s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2671 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4012.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:51,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:30,759] INFO : Step 7600/50000 rank0 | Train Loss: 9.5990 | Train Loss Unweighted: 3.1519 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.8134 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1443 | Regression_eta: 0.0105 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9177 | Classification_binary: 2.8134 | Regression_cos_phi: 0.6387 | Regression_energy: 1.9948 | Regression_eta: 0.5147 | Regression_pt: 2.1377 | Regression_sin_phi: 0.5819 | LR: 9.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0030 | max_backward: 0.2027 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3617.7 | uss_MB: 2834.0 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:07:30,973] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:10,588] INFO : Step 7700/50000 rank0 | Train Loss: 9.9504 | Train Loss Unweighted: 3.2750 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9115 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1574 | Regression_eta: 0.0096 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9373 | Classification_binary: 2.9115 | Regression_cos_phi: 0.5974 | Regression_energy: 2.1772 | Regression_eta: 0.4711 | Regression_pt: 2.3503 | Regression_sin_phi: 0.5056 | LR: 9.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4617s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0164 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3477.2 | uss_MB: 2691.9 | child_rss_MB: 4007.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:08:10,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:50,550] INFO : Step 7800/50000 rank0 | Train Loss: 10.2935 | Train Loss Unweighted: 3.2838 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.9211 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1529 | Regression_eta: 0.0129 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0429 | Classification_binary: 2.9211 | Regression_cos_phi: 0.7729 | Regression_energy: 2.1140 | Regression_eta: 0.6363 | Regression_pt: 2.2717 | Regression_sin_phi: 0.5347 | LR: 9.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4623s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1838 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2997 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3477.2 | uss_MB: 2692.0 | child_rss_MB: 4007.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:08:50,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:30,739] INFO : Step 7900/50000 rank0 | Train Loss: 10.5627 | Train Loss Unweighted: 3.2527 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1671 | Regression_eta: 0.0106 | Regression_pt: 0.1661 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9675 | Classification_binary: 2.8631 | Regression_cos_phi: 0.7461 | Regression_energy: 2.3113 | Regression_eta: 0.5228 | Regression_pt: 2.4988 | Regression_sin_phi: 0.6531 | LR: 9.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0495 | max_backward: 0.2035 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3477.2 | uss_MB: 2692.2 | child_rss_MB: 4012.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:09:30,953] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:10,968] INFO : Step 8000/50000 rank0 | Train Loss: 9.7712 | Train Loss Unweighted: 3.2109 | Train Loss Components: Classification: 0.0258 | Classification_binary: 2.8568 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1534 | Regression_eta: 0.0096 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0807 | Classification_binary: 2.8568 | Regression_cos_phi: 0.5271 | Regression_energy: 2.1208 | Regression_eta: 0.4720 | Regression_pt: 2.2444 | Regression_sin_phi: 0.4694 | LR: 9.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5226s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0938 | max_backward: 0.2027 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3477.2 | uss_MB: 2692.2 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:10:11,179] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:10:11,250] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 12:10:11,425] INFO : Running validation on rank0 +rank0 [2026-10-28 12:10:11,425] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:10:11,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:10:12,716] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 12:10:12,860] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:10:12,989] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:10:13,061] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:10:13,129] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:10:13,196] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:10:13,264] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:10:13,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:10:13,442] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:10:13,571] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:10:13,700] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:10:13,769] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:10:13,837] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:10:13,904] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:10:13,972] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:10:14,041] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:10:14,158] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:10:14,165] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:10:14,168] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:10:14,171] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:10:14,172] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:10:14,175] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:10:14,177] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:10:14,178] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:10:14,181] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:10:14,183] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:10:14,184] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:10:14,187] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:10:14,189] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:10:14,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:10:14,193] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:10:14,195] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:10:21,628] INFO : VALIDATION | Step=8000/50000 | Train Loss=9.7712 | Valid Loss=9.9230 | Train Loss Unweighted=3.2109 | Valid Loss Unweighted=3.2146 | Stale=0 | ETA=303.5m +rank0 [2026-10-28 12:10:21,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:21,699] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:10:21,702] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:21,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:23,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:24,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:24,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:25,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:25,748] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:25,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:25,807] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:10:25,810] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:25,909] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:27,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:28,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:29,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:29,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:29,840] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:10:29,843] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:29,896] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:10:29,899] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:10:30,053] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:31,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:32,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:33,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:33,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:33,924] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:33,931] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:34,183] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:34,630] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:36,498] INFO : Plotted jets +rank0 [2026-10-28 12:10:37,599] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:37,633] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:37,853] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:38,226] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:39,597] INFO : Plotted jets +rank0 [2026-10-28 12:10:40,653] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:40,688] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 12:10:40,890] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 12:10:41,302] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:42,679] INFO : Plotted jets +rank0 [2026-10-28 12:10:43,764] INFO : Plotted jet ratio +rank0 [2026-10-28 12:11:23,948] INFO : Step 8100/50000 rank0 | Train Loss: 9.6271 | Train Loss Unweighted: 3.1567 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.8186 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1419 | Regression_eta: 0.0098 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9971 | Classification_binary: 2.8186 | Regression_cos_phi: 0.6290 | Regression_energy: 1.9623 | Regression_eta: 0.4817 | Regression_pt: 2.1477 | Regression_sin_phi: 0.5905 | LR: 9.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3097 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3711.3 | uss_MB: 2926.8 | child_rss_MB: 4006.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:11:24,163] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:04,051] INFO : Step 8200/50000 rank0 | Train Loss: 9.4943 | Train Loss Unweighted: 3.2026 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.8705 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1393 | Regression_eta: 0.0102 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 2.8705 | Regression_cos_phi: 0.5838 | Regression_energy: 1.9267 | Regression_eta: 0.4989 | Regression_pt: 2.1356 | Regression_sin_phi: 0.5345 | LR: 9.42e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5575s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3078 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3713 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3711.3 | uss_MB: 2926.8 | child_rss_MB: 4008.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:12:04,265] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:44,118] INFO : Step 8300/50000 rank0 | Train Loss: 9.6151 | Train Loss Unweighted: 3.1340 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7764 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1550 | Regression_eta: 0.0096 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9320 | Classification_binary: 2.7764 | Regression_cos_phi: 0.5352 | Regression_energy: 2.1440 | Regression_eta: 0.4718 | Regression_pt: 2.3369 | Regression_sin_phi: 0.4188 | LR: 9.40e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3080 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:12:44,331] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:23,716] INFO : Step 8400/50000 rank0 | Train Loss: 9.9661 | Train Loss Unweighted: 3.1275 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7585 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1602 | Regression_eta: 0.0109 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9107 | Classification_binary: 2.7585 | Regression_cos_phi: 0.6547 | Regression_energy: 2.2157 | Regression_eta: 0.5365 | Regression_pt: 2.3614 | Regression_sin_phi: 0.5286 | LR: 9.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5304s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.2631 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4005.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:13:23,928] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:03,834] INFO : Step 8500/50000 rank0 | Train Loss: 9.6554 | Train Loss Unweighted: 3.1441 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8051 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1423 | Regression_eta: 0.0100 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0492 | Classification_binary: 2.8051 | Regression_cos_phi: 0.6497 | Regression_energy: 1.9676 | Regression_eta: 0.4899 | Regression_pt: 2.1397 | Regression_sin_phi: 0.5542 | LR: 9.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3109 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3985 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:04,047] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:43,930] INFO : Step 8600/50000 rank0 | Train Loss: 9.4448 | Train Loss Unweighted: 3.1686 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.8290 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1458 | Regression_eta: 0.0090 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9319 | Classification_binary: 2.8290 | Regression_cos_phi: 0.5732 | Regression_energy: 2.0166 | Regression_eta: 0.4444 | Regression_pt: 2.1954 | Regression_sin_phi: 0.4542 | LR: 9.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0336 | max_backward: 0.2034 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4009.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:44,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:24,217] INFO : Step 8700/50000 rank0 | Train Loss: 9.1874 | Train Loss Unweighted: 2.9863 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.6668 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1338 | Regression_eta: 0.0098 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9925 | Classification_binary: 2.6668 | Regression_cos_phi: 0.6248 | Regression_energy: 1.8497 | Regression_eta: 0.4811 | Regression_pt: 1.9970 | Regression_sin_phi: 0.5755 | LR: 9.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2652 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3870 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3553.4 | uss_MB: 2767.8 | child_rss_MB: 4010.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:24,430] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:04,526] INFO : Step 8800/50000 rank0 | Train Loss: 10.2198 | Train Loss Unweighted: 3.2391 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.8778 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1509 | Regression_eta: 0.0131 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0200 | Classification_binary: 2.8778 | Regression_cos_phi: 0.7283 | Regression_energy: 2.0869 | Regression_eta: 0.6432 | Regression_pt: 2.2839 | Regression_sin_phi: 0.5797 | LR: 9.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1227 | max_loss: 0.0031 | max_backward: 0.2032 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3553.4 | uss_MB: 2768.0 | child_rss_MB: 4003.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:16:04,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:43,871] INFO : Step 8900/50000 rank0 | Train Loss: 10.0898 | Train Loss Unweighted: 3.1272 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7541 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1605 | Regression_eta: 0.0100 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0216 | Classification_binary: 2.7541 | Regression_cos_phi: 0.5974 | Regression_energy: 2.2188 | Regression_eta: 0.4937 | Regression_pt: 2.3813 | Regression_sin_phi: 0.6229 | LR: 9.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0792 | max_backward: 0.2029 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3553.4 | uss_MB: 2768.2 | child_rss_MB: 4002.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:16:44,086] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:23,203] INFO : Step 9000/50000 rank0 | Train Loss: 9.0820 | Train Loss Unweighted: 3.0574 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7379 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1351 | Regression_eta: 0.0093 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9470 | Classification_binary: 2.7379 | Regression_cos_phi: 0.5842 | Regression_energy: 1.8682 | Regression_eta: 0.4586 | Regression_pt: 2.0451 | Regression_sin_phi: 0.4411 | LR: 9.30e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5917s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3113 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3553.4 | uss_MB: 2768.0 | child_rss_MB: 4005.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:17:23,416] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:23,440] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 12:17:23,721] INFO : Running validation on rank0 +rank0 [2026-10-28 12:17:23,721] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:17:23,721] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:17:24,906] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 12:17:25,049] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:17:25,181] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:17:25,254] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:17:25,322] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:17:25,390] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:17:25,458] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:17:25,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:17:25,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:17:25,768] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:17:25,899] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:17:25,970] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:17:26,038] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:17:26,105] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:17:26,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:17:26,240] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:17:26,344] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:17:26,449] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:17:26,452] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:17:26,454] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:17:26,457] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:17:26,458] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:17:26,461] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:17:26,464] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:17:26,465] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:17:26,467] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:17:26,468] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:17:26,471] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:17:26,473] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:17:26,474] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:17:26,477] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:17:26,479] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:17:33,808] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.0820 | Valid Loss=9.7088 | Train Loss Unweighted=3.0574 | Valid Loss Unweighted=3.1556 | Stale=0 | ETA=296.1m +rank0 [2026-10-28 12:17:33,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:33,880] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:17:33,884] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:34,072] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:36,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:36,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:37,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:38,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:38,164] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:17:38,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:38,267] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:17:38,286] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:38,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:40,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:40,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:41,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:42,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:42,284] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:17:42,287] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:42,365] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:17:42,368] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:17:42,522] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:44,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:44,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:45,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:46,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:46,393] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:17:46,400] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:46,656] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:47,103] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:48,991] INFO : Plotted jets +rank0 [2026-10-28 12:17:50,087] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:50,120] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:50,345] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:50,717] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:52,089] INFO : Plotted jets +rank0 [2026-10-28 12:17:53,204] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:53,238] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:53,442] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:53,865] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:55,251] INFO : Plotted jets +rank0 [2026-10-28 12:17:56,350] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:36,809] INFO : Step 9100/50000 rank0 | Train Loss: 9.4462 | Train Loss Unweighted: 3.1075 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7681 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1459 | Regression_eta: 0.0099 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9149 | Classification_binary: 2.7681 | Regression_cos_phi: 0.5806 | Regression_energy: 2.0180 | Regression_eta: 0.4850 | Regression_pt: 2.1696 | Regression_sin_phi: 0.5099 | LR: 9.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5157s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1865 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3624.4 | uss_MB: 2838.1 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:18:37,031] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:16,738] INFO : Step 9200/50000 rank0 | Train Loss: 9.6513 | Train Loss Unweighted: 3.0969 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7428 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1519 | Regression_eta: 0.0098 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9394 | Classification_binary: 2.7428 | Regression_cos_phi: 0.5857 | Regression_energy: 2.1003 | Regression_eta: 0.4818 | Regression_pt: 2.2911 | Regression_sin_phi: 0.5103 | LR: 9.27e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5686s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3045 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3626.4 | uss_MB: 2839.5 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:19:16,950] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:56,763] INFO : Step 9300/50000 rank0 | Train Loss: 8.9964 | Train Loss Unweighted: 3.0644 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7522 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1330 | Regression_eta: 0.0093 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9222 | Classification_binary: 2.7522 | Regression_cos_phi: 0.5767 | Regression_energy: 1.8386 | Regression_eta: 0.4574 | Regression_pt: 1.9691 | Regression_sin_phi: 0.4803 | LR: 9.25e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4845s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3050 | max_optimizer_step: 0.1218 | max_tensorboard_logging: 0.2727 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3627.4 | uss_MB: 2839.7 | child_rss_MB: 4022.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:19:56,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:37,252] INFO : Step 9400/50000 rank0 | Train Loss: 9.4310 | Train Loss Unweighted: 3.1401 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8116 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1394 | Regression_eta: 0.0100 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9731 | Classification_binary: 2.8116 | Regression_cos_phi: 0.6445 | Regression_energy: 1.9273 | Regression_eta: 0.4931 | Regression_pt: 2.0622 | Regression_sin_phi: 0.5193 | LR: 9.24e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3097 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.3924 | max_tensorboard_flush: 0.0073 | Memory: rss_MB: 3627.4 | uss_MB: 2839.8 | child_rss_MB: 4000.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:20:37,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:17,040] INFO : Step 9500/50000 rank0 | Train Loss: 9.5144 | Train Loss Unweighted: 3.1366 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7996 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1445 | Regression_eta: 0.0101 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9606 | Classification_binary: 2.7996 | Regression_cos_phi: 0.5773 | Regression_energy: 1.9982 | Regression_eta: 0.4975 | Regression_pt: 2.1212 | Regression_sin_phi: 0.5600 | LR: 9.22e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0429 | max_forward: 0.0779 | max_loss: 0.0031 | max_backward: 0.2035 | max_optimizer_step: 0.1839 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3627.4 | uss_MB: 2839.9 | child_rss_MB: 4000.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:21:17,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:56,949] INFO : Step 9600/50000 rank0 | Train Loss: 10.0864 | Train Loss Unweighted: 3.1032 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.7340 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1577 | Regression_eta: 0.0105 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0197 | Classification_binary: 2.7340 | Regression_cos_phi: 0.7017 | Regression_energy: 2.1807 | Regression_eta: 0.5170 | Regression_pt: 2.3444 | Regression_sin_phi: 0.5890 | LR: 9.21e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4876s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1840 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3627.4 | uss_MB: 2839.9 | child_rss_MB: 4003.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:21:57,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:37,905] INFO : Step 9700/50000 rank0 | Train Loss: 9.7767 | Train Loss Unweighted: 3.1829 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8360 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1450 | Regression_eta: 0.0104 | Regression_pt: 0.1489 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9771 | Classification_binary: 2.8360 | Regression_cos_phi: 0.7263 | Regression_energy: 2.0049 | Regression_eta: 0.5088 | Regression_pt: 2.2406 | Regression_sin_phi: 0.4830 | LR: 9.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5840s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0714 | max_backward: 0.2029 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.4343 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3626.5 | uss_MB: 2838.0 | child_rss_MB: 4003.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:22:38,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:17,637] INFO : Step 9800/50000 rank0 | Train Loss: 10.3028 | Train Loss Unweighted: 3.2484 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8711 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1620 | Regression_eta: 0.0110 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 2.8711 | Regression_cos_phi: 0.6186 | Regression_energy: 2.2396 | Regression_eta: 0.5396 | Regression_pt: 2.4138 | Regression_sin_phi: 0.6289 | LR: 9.17e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4714s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3066 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2851 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3627.5 | uss_MB: 2839.5 | child_rss_MB: 4032.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:23:17,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:57,269] INFO : Step 9900/50000 rank0 | Train Loss: 9.7808 | Train Loss Unweighted: 3.1641 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.8155 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1475 | Regression_eta: 0.0101 | Regression_pt: 0.1477 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0065 | Classification_binary: 2.8155 | Regression_cos_phi: 0.6915 | Regression_energy: 2.0391 | Regression_eta: 0.4976 | Regression_pt: 2.2217 | Regression_sin_phi: 0.5089 | LR: 9.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0034 | max_backward: 0.3087 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3627.5 | uss_MB: 2839.7 | child_rss_MB: 4006.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:23:57,482] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:37,108] INFO : Step 10000/50000 rank0 | Train Loss: 11.4549 | Train Loss Unweighted: 3.3507 | Train Loss Components: Classification: 0.0263 | Classification_binary: 2.9295 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1760 | Regression_eta: 0.0176 | Regression_pt: 0.1776 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1037 | Classification_binary: 2.9295 | Regression_cos_phi: 0.7199 | Regression_energy: 2.4339 | Regression_eta: 0.8671 | Regression_pt: 2.6723 | Regression_sin_phi: 0.7285 | LR: 9.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.2059 | max_loss: 0.0031 | max_backward: 0.2871 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3627.5 | uss_MB: 2842.2 | child_rss_MB: 4009.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:24:37,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:37,354] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 12:24:37,525] INFO : Running validation on rank0 +rank0 [2026-10-28 12:24:37,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:24:37,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:24:38,798] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 12:24:38,942] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:24:39,073] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:24:39,147] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:24:39,215] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:24:39,283] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:24:39,353] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:24:39,422] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:24:39,532] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:24:39,660] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:24:39,790] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:24:39,860] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:24:39,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:24:39,995] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:24:40,064] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:24:40,133] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:24:40,235] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:24:40,242] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:24:40,245] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:24:40,248] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:24:40,249] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:24:40,251] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:24:40,254] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:24:40,255] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:24:40,258] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:24:40,259] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:24:40,261] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:24:40,264] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:24:40,265] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:24:40,268] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:24:40,271] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:24:40,272] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:24:47,561] INFO : VALIDATION | Step=10000/50000 | Train Loss=11.4549 | Valid Loss=9.6757 | Train Loss Unweighted=3.3507 | Valid Loss Unweighted=3.1405 | Stale=0 | ETA=288.9m +rank0 [2026-10-28 12:24:47,570] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:47,661] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:24:47,664] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:47,813] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:49,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:50,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:51,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:51,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:51,872] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:24:51,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:51,932] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:24:51,935] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:52,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:53,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:54,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:55,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:55,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:55,887] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:24:55,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:55,970] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:24:55,973] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:24:56,131] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:57,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:58,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:59,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:59,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:00,025] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:25:00,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:25:00,282] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:25:00,727] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:02,632] INFO : Plotted jets +rank0 [2026-10-28 12:25:03,717] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:03,750] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:03,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:04,365] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:05,734] INFO : Plotted jets +rank0 [2026-10-28 12:25:06,842] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:06,875] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 12:25:07,074] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 12:25:07,483] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:08,863] INFO : Plotted jets +rank0 [2026-10-28 12:25:09,949] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:50,559] INFO : Step 10100/50000 rank0 | Train Loss: 10.1513 | Train Loss Unweighted: 3.2501 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8889 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1526 | Regression_eta: 0.0120 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9938 | Classification_binary: 2.8889 | Regression_cos_phi: 0.6849 | Regression_energy: 2.1099 | Regression_eta: 0.5900 | Regression_pt: 2.2906 | Regression_sin_phi: 0.5933 | LR: 9.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1854 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2995 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3700.5 | uss_MB: 2912.6 | child_rss_MB: 4002.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:25:50,777] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:29,892] INFO : Step 10200/50000 rank0 | Train Loss: 9.4393 | Train Loss Unweighted: 3.0790 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7480 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1401 | Regression_eta: 0.0107 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9575 | Classification_binary: 2.7480 | Regression_cos_phi: 0.6749 | Regression_energy: 1.9367 | Regression_eta: 0.5279 | Regression_pt: 2.0791 | Regression_sin_phi: 0.5152 | LR: 9.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4883s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3700.5 | uss_MB: 2913.7 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:26:30,110] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:09,559] INFO : Step 10300/50000 rank0 | Train Loss: 9.2976 | Train Loss Unweighted: 3.1214 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7827 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1455 | Regression_eta: 0.0081 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9449 | Classification_binary: 2.7827 | Regression_cos_phi: 0.4745 | Regression_energy: 2.0122 | Regression_eta: 0.3993 | Regression_pt: 2.2129 | Regression_sin_phi: 0.4710 | LR: 9.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5581s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0911 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3701.5 | uss_MB: 2914.0 | child_rss_MB: 4006.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:27:09,781] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:49,400] INFO : Step 10400/50000 rank0 | Train Loss: 9.9020 | Train Loss Unweighted: 3.1716 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.8254 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1487 | Regression_eta: 0.0127 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9523 | Classification_binary: 2.8254 | Regression_cos_phi: 0.7748 | Regression_energy: 2.0568 | Regression_eta: 0.6248 | Regression_pt: 2.1174 | Regression_sin_phi: 0.5505 | LR: 9.07e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4032s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1093 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3701.5 | uss_MB: 2914.2 | child_rss_MB: 4007.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:27:49,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:29,708] INFO : Step 10500/50000 rank0 | Train Loss: 9.2844 | Train Loss Unweighted: 3.0615 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7291 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1428 | Regression_eta: 0.0090 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9802 | Classification_binary: 2.7291 | Regression_cos_phi: 0.6084 | Regression_energy: 1.9752 | Regression_eta: 0.4427 | Regression_pt: 2.1118 | Regression_sin_phi: 0.4370 | LR: 9.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5190s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0989 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 0.2736 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3701.5 | uss_MB: 2914.3 | child_rss_MB: 4005.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:28:29,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:09,316] INFO : Step 10600/50000 rank0 | Train Loss: 8.9772 | Train Loss Unweighted: 3.0284 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7074 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1383 | Regression_eta: 0.0094 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8634 | Classification_binary: 2.7074 | Regression_cos_phi: 0.5437 | Regression_energy: 1.9123 | Regression_eta: 0.4636 | Regression_pt: 2.0630 | Regression_sin_phi: 0.4239 | LR: 9.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0475 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.2681 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3701.5 | uss_MB: 2914.3 | child_rss_MB: 4001.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:29:09,545] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:49,495] INFO : Step 10700/50000 rank0 | Train Loss: 8.6186 | Train Loss Unweighted: 2.9534 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6556 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1277 | Regression_eta: 0.0111 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8309 | Classification_binary: 2.6556 | Regression_cos_phi: 0.5287 | Regression_energy: 1.7657 | Regression_eta: 0.5458 | Regression_pt: 1.8607 | Regression_sin_phi: 0.4311 | LR: 9.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4867s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3101 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3699.6 | uss_MB: 2912.3 | child_rss_MB: 4023.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:29:49,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:28,880] INFO : Step 10800/50000 rank0 | Train Loss: 9.1274 | Train Loss Unweighted: 2.9811 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.6519 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1405 | Regression_eta: 0.0082 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9284 | Classification_binary: 2.6519 | Regression_cos_phi: 0.5314 | Regression_energy: 1.9423 | Regression_eta: 0.4039 | Regression_pt: 2.1178 | Regression_sin_phi: 0.5516 | LR: 9.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3940s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1312 | max_loss: 0.0031 | max_backward: 0.2025 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3700.6 | uss_MB: 2913.9 | child_rss_MB: 4004.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:30:29,094] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:08,706] INFO : Step 10900/50000 rank0 | Train Loss: 9.3713 | Train Loss Unweighted: 3.0842 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.7456 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1457 | Regression_eta: 0.0097 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9973 | Classification_binary: 2.7456 | Regression_cos_phi: 0.5253 | Regression_energy: 2.0142 | Regression_eta: 0.4780 | Regression_pt: 2.1620 | Regression_sin_phi: 0.4489 | LR: 8.99e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3093 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.2736 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3701.6 | uss_MB: 2914.2 | child_rss_MB: 4026.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:31:08,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:48,890] INFO : Step 11000/50000 rank0 | Train Loss: 9.4699 | Train Loss Unweighted: 3.0787 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7430 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1429 | Regression_eta: 0.0107 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9585 | Classification_binary: 2.7430 | Regression_cos_phi: 0.6423 | Regression_energy: 1.9757 | Regression_eta: 0.5270 | Regression_pt: 2.1175 | Regression_sin_phi: 0.5059 | LR: 8.97e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.4522s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0031 | max_backward: 0.3103 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3701.6 | uss_MB: 2914.2 | child_rss_MB: 4003.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 12:31:49,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:49,122] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 12:31:49,301] INFO : Running validation on rank0 +rank0 [2026-10-28 12:31:49,301] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:31:49,301] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:31:50,572] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 12:31:50,714] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:31:50,844] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:31:50,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:31:50,984] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:31:51,052] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:31:51,121] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:31:51,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:31:51,300] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:31:51,428] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:31:51,558] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:31:51,627] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:31:51,696] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:31:51,763] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:31:51,832] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:31:51,901] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:31:52,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:31:52,018] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:31:52,021] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:31:52,024] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:31:52,025] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:31:52,027] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:31:52,030] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:31:52,031] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:31:52,033] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:31:52,036] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:31:52,037] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:31:52,039] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:31:52,042] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:31:52,043] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:31:52,045] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:31:52,048] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:31:59,426] INFO : VALIDATION | Step=11000/50000 | Train Loss=9.4699 | Valid Loss=9.5823 | Train Loss Unweighted=3.0787 | Valid Loss Unweighted=3.1123 | Stale=0 | ETA=281.6m +rank0 [2026-10-28 12:31:59,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:59,522] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:31:59,525] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:59,677] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:01,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:02,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:02,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:03,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:03,650] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:32:03,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:03,741] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:32:03,758] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:03,915] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:05,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:06,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:06,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:07,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:07,718] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:07,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:07,794] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:32:07,797] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:32:07,954] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:09,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:10,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:11,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:11,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:11,816] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:11,823] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:32:12,075] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:32:12,523] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:14,440] INFO : Plotted jets +rank0 [2026-10-28 12:32:15,529] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:15,562] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:15,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:16,160] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:17,528] INFO : Plotted jets +rank0 [2026-10-28 12:32:18,641] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:18,674] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 12:32:18,879] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 12:32:19,289] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:20,674] INFO : Plotted jets +rank0 [2026-10-28 12:32:21,764] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:02,364] INFO : Step 11100/50000 rank0 | Train Loss: 9.3667 | Train Loss Unweighted: 3.0498 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7210 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1408 | Regression_eta: 0.0100 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9418 | Classification_binary: 2.7210 | Regression_cos_phi: 0.7342 | Regression_energy: 1.9469 | Regression_eta: 0.4929 | Regression_pt: 2.0465 | Regression_sin_phi: 0.4834 | LR: 8.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4930s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0550 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.2032 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3773.1 | uss_MB: 2987.9 | child_rss_MB: 4003.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:02,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:42,728] INFO : Step 11200/50000 rank0 | Train Loss: 9.3205 | Train Loss Unweighted: 3.0531 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1381 | Regression_eta: 0.0108 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0345 | Classification_binary: 2.7267 | Regression_cos_phi: 0.5827 | Regression_energy: 1.9093 | Regression_eta: 0.5295 | Regression_pt: 2.0352 | Regression_sin_phi: 0.5026 | LR: 8.93e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3054 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3774.1 | uss_MB: 2989.0 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:42,940] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:23,138] INFO : Step 11300/50000 rank0 | Train Loss: 9.4410 | Train Loss Unweighted: 3.0109 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6734 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1457 | Regression_eta: 0.0124 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9131 | Classification_binary: 2.6734 | Regression_cos_phi: 0.6360 | Regression_energy: 2.0152 | Regression_eta: 0.6094 | Regression_pt: 2.0951 | Regression_sin_phi: 0.4988 | LR: 8.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0491 | max_backward: 0.2030 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3654 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3774.1 | uss_MB: 2989.2 | child_rss_MB: 4002.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:34:23,353] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:02,789] INFO : Step 11400/50000 rank0 | Train Loss: 8.9424 | Train Loss Unweighted: 3.0045 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7113 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1192 | Regression_eta: 0.0126 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8943 | Classification_binary: 2.7113 | Regression_cos_phi: 0.6945 | Regression_energy: 1.6489 | Regression_eta: 0.6203 | Regression_pt: 1.7963 | Regression_sin_phi: 0.5768 | LR: 8.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1754 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3774.1 | uss_MB: 2989.3 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:35:03,006] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:42,875] INFO : Step 11500/50000 rank0 | Train Loss: 9.8073 | Train Loss Unweighted: 2.9903 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.6407 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1473 | Regression_eta: 0.0158 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9572 | Classification_binary: 2.6407 | Regression_cos_phi: 0.6854 | Regression_energy: 2.0375 | Regression_eta: 0.7761 | Regression_pt: 2.1633 | Regression_sin_phi: 0.5471 | LR: 8.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5447s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.1078 | max_backward: 0.2032 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3587 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3774.1 | uss_MB: 2989.3 | child_rss_MB: 4005.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:35:43,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:22,868] INFO : Step 11600/50000 rank0 | Train Loss: 9.5090 | Train Loss Unweighted: 3.0012 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.6543 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1483 | Regression_eta: 0.0109 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9943 | Classification_binary: 2.6543 | Regression_cos_phi: 0.5675 | Regression_energy: 2.0509 | Regression_eta: 0.5337 | Regression_pt: 2.2061 | Regression_sin_phi: 0.5022 | LR: 8.86e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3101 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3774.1 | uss_MB: 2989.4 | child_rss_MB: 4006.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:36:23,083] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:03,294] INFO : Step 11700/50000 rank0 | Train Loss: 9.9315 | Train Loss Unweighted: 3.1608 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.8222 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1434 | Regression_eta: 0.0143 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9602 | Classification_binary: 2.8222 | Regression_cos_phi: 0.7503 | Regression_energy: 1.9834 | Regression_eta: 0.7006 | Regression_pt: 2.0265 | Regression_sin_phi: 0.6883 | LR: 8.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2524 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3772.1 | uss_MB: 2987.8 | child_rss_MB: 4003.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:37:03,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:43,387] INFO : Step 11800/50000 rank0 | Train Loss: 10.3230 | Train Loss Unweighted: 3.1269 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7441 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1667 | Regression_eta: 0.0140 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8870 | Classification_binary: 2.7441 | Regression_cos_phi: 0.6702 | Regression_energy: 2.3051 | Regression_eta: 0.6875 | Regression_pt: 2.4071 | Regression_sin_phi: 0.6221 | LR: 8.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5684s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3090 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3773.1 | uss_MB: 2989.1 | child_rss_MB: 4003.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:43,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:23,511] INFO : Step 11900/50000 rank0 | Train Loss: 8.7285 | Train Loss Unweighted: 2.9571 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.6491 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1313 | Regression_eta: 0.0070 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9413 | Classification_binary: 2.6491 | Regression_cos_phi: 0.5704 | Regression_energy: 1.8162 | Regression_eta: 0.3421 | Regression_pt: 1.9711 | Regression_sin_phi: 0.4384 | LR: 8.80e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3092 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3773.1 | uss_MB: 2989.2 | child_rss_MB: 4002.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:38:23,726] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:03,153] INFO : Step 12000/50000 rank0 | Train Loss: 8.8581 | Train Loss Unweighted: 2.9674 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6506 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1360 | Regression_eta: 0.0091 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9102 | Classification_binary: 2.6506 | Regression_cos_phi: 0.5353 | Regression_energy: 1.8807 | Regression_eta: 0.4448 | Regression_pt: 2.0283 | Regression_sin_phi: 0.4082 | LR: 8.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3994s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1148 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3773.1 | uss_MB: 2989.2 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:39:03,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:03,387] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 12:39:03,564] INFO : Running validation on rank0 +rank0 [2026-10-28 12:39:03,565] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:39:03,565] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:39:04,853] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 12:39:04,998] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:39:05,126] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:39:05,200] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:39:05,268] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:39:05,335] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:39:05,404] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:39:05,472] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:39:05,584] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:39:05,713] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:39:05,844] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:39:05,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:39:05,982] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:39:06,049] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:39:06,117] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:39:06,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:39:06,295] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:39:06,302] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:39:06,305] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:39:06,308] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:39:06,309] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:39:06,312] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:39:06,315] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:39:06,315] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:39:06,318] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:39:06,319] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:39:06,322] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:39:06,324] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:39:06,327] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:39:06,329] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:39:06,331] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:39:06,332] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:39:13,676] INFO : VALIDATION | Step=12000/50000 | Train Loss=8.8581 | Valid Loss=9.3530 | Train Loss Unweighted=2.9674 | Valid Loss Unweighted=3.0414 | Stale=0 | ETA=274.5m +rank0 [2026-10-28 12:39:13,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:13,769] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:39:13,773] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:13,928] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:15,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:16,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:17,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:18,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:18,104] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:39:18,107] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:18,165] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:39:18,168] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:18,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:20,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:20,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:21,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:22,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:22,168] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:22,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:22,242] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:39:22,245] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:39:22,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:24,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:24,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:25,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:26,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:26,245] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:26,253] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:26,502] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:26,945] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:28,812] INFO : Plotted jets +rank0 [2026-10-28 12:39:29,897] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:29,929] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:30,153] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:30,526] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:31,895] INFO : Plotted jets +rank0 [2026-10-28 12:39:32,950] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:32,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 12:39:33,184] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 12:39:33,598] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:34,973] INFO : Plotted jets +rank0 [2026-10-28 12:39:36,060] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:16,493] INFO : Step 12100/50000 rank0 | Train Loss: 9.3868 | Train Loss Unweighted: 3.0250 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6867 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1458 | Regression_eta: 0.0103 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9622 | Classification_binary: 2.6867 | Regression_cos_phi: 0.5895 | Regression_energy: 2.0165 | Regression_eta: 0.5063 | Regression_pt: 2.1322 | Regression_sin_phi: 0.4934 | LR: 8.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0505 | max_backward: 0.2029 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3845.6 | uss_MB: 3062.1 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:40:16,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:56,180] INFO : Step 12200/50000 rank0 | Train Loss: 9.1465 | Train Loss Unweighted: 3.0782 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.7559 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1369 | Regression_eta: 0.0100 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9714 | Classification_binary: 2.7559 | Regression_cos_phi: 0.5194 | Regression_energy: 1.8938 | Regression_eta: 0.4907 | Regression_pt: 2.0492 | Regression_sin_phi: 0.4662 | LR: 8.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5143s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0853 | max_forward: 0.0780 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3846.6 | uss_MB: 3063.6 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:40:56,405] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:35,956] INFO : Step 12300/50000 rank0 | Train Loss: 9.4439 | Train Loss Unweighted: 3.0429 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6909 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1536 | Regression_eta: 0.0087 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9124 | Classification_binary: 2.6909 | Regression_cos_phi: 0.5679 | Regression_energy: 2.1235 | Regression_eta: 0.4260 | Regression_pt: 2.2830 | Regression_sin_phi: 0.4402 | LR: 8.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.1039 | max_backward: 0.2034 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3628 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3846.6 | uss_MB: 3063.6 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:41:36,170] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:42:16,247] INFO : Step 12400/50000 rank0 | Train Loss: 9.4438 | Train Loss Unweighted: 3.0491 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7214 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1356 | Regression_eta: 0.0168 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9023 | Classification_binary: 2.7214 | Regression_cos_phi: 0.5846 | Regression_energy: 1.8756 | Regression_eta: 0.8249 | Regression_pt: 2.0552 | Regression_sin_phi: 0.4798 | LR: 8.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.3626 | max_tensorboard_flush: 0.0262 | Memory: rss_MB: 3846.6 | uss_MB: 3063.7 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:42:16,465] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:55,550] INFO : Step 12500/50000 rank0 | Train Loss: 10.0017 | Train Loss Unweighted: 3.0837 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.7101 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1619 | Regression_eta: 0.0110 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9616 | Classification_binary: 2.7101 | Regression_cos_phi: 0.6415 | Regression_energy: 2.2383 | Regression_eta: 0.5415 | Regression_pt: 2.3943 | Regression_sin_phi: 0.5143 | LR: 8.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4218s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1333 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3846.6 | uss_MB: 3063.8 | child_rss_MB: 4003.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:42:55,766] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:35,908] INFO : Step 12600/50000 rank0 | Train Loss: 9.4728 | Train Loss Unweighted: 2.9968 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6558 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1465 | Regression_eta: 0.0133 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8776 | Classification_binary: 2.6558 | Regression_cos_phi: 0.5718 | Regression_energy: 2.0255 | Regression_eta: 0.6545 | Regression_pt: 2.1382 | Regression_sin_phi: 0.5494 | LR: 8.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4566s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0377 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.2028 | max_optimizer_step: 0.1648 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3846.6 | uss_MB: 3063.8 | child_rss_MB: 4006.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:43:36,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:15,980] INFO : Step 12700/50000 rank0 | Train Loss: 9.3671 | Train Loss Unweighted: 2.9971 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6666 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1421 | Regression_eta: 0.0169 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8436 | Classification_binary: 2.6666 | Regression_cos_phi: 0.5186 | Regression_energy: 1.9655 | Regression_eta: 0.8292 | Regression_pt: 2.0238 | Regression_sin_phi: 0.5198 | LR: 8.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4347s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1324 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2787 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3845.5 | uss_MB: 3062.1 | child_rss_MB: 4004.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:44:16,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:56,030] INFO : Step 12800/50000 rank0 | Train Loss: 9.0291 | Train Loss Unweighted: 3.0914 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7666 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1407 | Regression_eta: 0.0083 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9605 | Classification_binary: 2.7666 | Regression_cos_phi: 0.4561 | Regression_energy: 1.9461 | Regression_eta: 0.4075 | Regression_pt: 2.0920 | Regression_sin_phi: 0.4003 | LR: 8.62e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6118s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0780 | max_backward: 0.2027 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.4290 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 3847.5 | uss_MB: 3063.6 | child_rss_MB: 4004.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:44:56,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:35,981] INFO : Step 12900/50000 rank0 | Train Loss: 9.2413 | Train Loss Unweighted: 3.0749 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7472 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1396 | Regression_eta: 0.0093 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9096 | Classification_binary: 2.7472 | Regression_cos_phi: 0.5819 | Regression_energy: 1.9303 | Regression_eta: 0.4566 | Regression_pt: 2.0954 | Regression_sin_phi: 0.5204 | LR: 8.60e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4976s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0668 | max_backward: 0.2032 | max_optimizer_step: 0.1153 | max_tensorboard_logging: 0.3561 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3847.5 | uss_MB: 3063.7 | child_rss_MB: 4006.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:45:36,197] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:16,187] INFO : Step 13000/50000 rank0 | Train Loss: 9.9072 | Train Loss Unweighted: 3.1570 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.7955 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1558 | Regression_eta: 0.0112 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9497 | Classification_binary: 2.7955 | Regression_cos_phi: 0.5807 | Regression_energy: 2.1546 | Regression_eta: 0.5524 | Regression_pt: 2.3024 | Regression_sin_phi: 0.5718 | LR: 8.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4636s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1819 | max_loss: 0.0033 | max_backward: 0.2036 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3847.5 | uss_MB: 3063.7 | child_rss_MB: 4003.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:46:16,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:16,434] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 12:46:16,606] INFO : Running validation on rank0 +rank0 [2026-10-28 12:46:16,607] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:46:16,607] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:46:17,889] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 12:46:18,034] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:46:18,163] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:46:18,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:46:18,305] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:46:18,372] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:46:18,441] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:46:18,510] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:46:18,620] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:46:18,749] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:46:18,879] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:46:18,949] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:46:19,017] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:46:19,084] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:46:19,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:46:19,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:46:19,328] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:46:19,335] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:46:19,338] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:46:19,341] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:46:19,342] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:46:19,344] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:46:19,347] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:46:19,348] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:46:19,349] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:46:19,351] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:46:19,354] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:46:19,355] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:46:19,357] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:46:19,360] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:46:19,361] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:46:19,363] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:46:26,645] INFO : VALIDATION | Step=13000/50000 | Train Loss=9.9072 | Valid Loss=9.3069 | Train Loss Unweighted=3.1570 | Valid Loss Unweighted=3.0420 | Stale=0 | ETA=267.2m +rank0 [2026-10-28 12:46:26,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:26,720] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:46:26,723] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:26,840] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:28,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:29,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:30,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:30,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:30,941] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:30,945] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:31,020] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:46:31,022] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:31,104] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:32,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:33,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:34,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:34,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:35,075] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:35,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:35,157] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:46:35,160] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:46:35,235] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:37,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:37,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:38,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:39,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:39,175] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:39,183] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:39,431] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:39,876] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:41,790] INFO : Plotted jets +rank0 [2026-10-28 12:46:42,882] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:42,916] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:43,138] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:43,509] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:44,876] INFO : Plotted jets +rank0 [2026-10-28 12:46:45,983] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:46,015] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 12:46:46,217] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 12:46:46,628] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:48,007] INFO : Plotted jets +rank0 [2026-10-28 12:46:49,091] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:29,303] INFO : Step 13100/50000 rank0 | Train Loss: 9.3780 | Train Loss Unweighted: 2.9958 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6629 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1401 | Regression_eta: 0.0099 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9140 | Classification_binary: 2.6629 | Regression_cos_phi: 0.6304 | Regression_energy: 1.9371 | Regression_eta: 0.4864 | Regression_pt: 2.1141 | Regression_sin_phi: 0.6330 | LR: 8.56e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5847s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.1040 | max_backward: 0.2033 | max_optimizer_step: 0.1204 | max_tensorboard_logging: 0.3471 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4069.6 | uss_MB: 3287.5 | child_rss_MB: 4002.3 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:47:29,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:09,301] INFO : Step 13200/50000 rank0 | Train Loss: 8.7494 | Train Loss Unweighted: 2.9192 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6135 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1308 | Regression_eta: 0.0096 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8338 | Classification_binary: 2.6135 | Regression_cos_phi: 0.5659 | Regression_energy: 1.8086 | Regression_eta: 0.4725 | Regression_pt: 1.9170 | Regression_sin_phi: 0.5381 | LR: 8.54e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2562 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3921.5 | uss_MB: 3138.0 | child_rss_MB: 4007.3 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:48:09,513] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,083] INFO : Step 13300/50000 rank0 | Train Loss: 9.5944 | Train Loss Unweighted: 3.0840 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7284 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1536 | Regression_eta: 0.0095 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9386 | Classification_binary: 2.7284 | Regression_cos_phi: 0.5499 | Regression_energy: 2.1235 | Regression_eta: 0.4667 | Regression_pt: 2.3093 | Regression_sin_phi: 0.4780 | LR: 8.52e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1848 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3921.5 | uss_MB: 3137.6 | child_rss_MB: 4003.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:48:49,299] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:29,538] INFO : Step 13400/50000 rank0 | Train Loss: 9.5045 | Train Loss Unweighted: 3.0889 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7552 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1416 | Regression_eta: 0.0108 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9590 | Classification_binary: 2.7552 | Regression_cos_phi: 0.7123 | Regression_energy: 1.9578 | Regression_eta: 0.5299 | Regression_pt: 2.0921 | Regression_sin_phi: 0.4981 | LR: 8.50e-04 | DataLoad Time: 0.0034s | Model Forward Time: 1.4848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.3100 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.2708 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3921.5 | uss_MB: 3137.6 | child_rss_MB: 4034.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:49:29,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:09,108] INFO : Step 13500/50000 rank0 | Train Loss: 9.4437 | Train Loss Unweighted: 3.0760 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7339 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1472 | Regression_eta: 0.0098 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9085 | Classification_binary: 2.7339 | Regression_cos_phi: 0.5700 | Regression_energy: 2.0350 | Regression_eta: 0.4823 | Regression_pt: 2.1930 | Regression_sin_phi: 0.5210 | LR: 8.48e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8302s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1055 | max_forward: 0.1046 | max_loss: 0.0033 | max_backward: 0.3950 | max_optimizer_step: 0.1200 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0035 | Memory: rss_MB: 3921.5 | uss_MB: 3140.2 | child_rss_MB: 4006.9 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:50:09,322] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:48,790] INFO : Step 13600/50000 rank0 | Train Loss: 9.8253 | Train Loss Unweighted: 3.0410 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6743 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1588 | Regression_eta: 0.0104 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8788 | Classification_binary: 2.6743 | Regression_cos_phi: 0.6764 | Regression_energy: 2.1954 | Regression_eta: 0.5108 | Regression_pt: 2.3663 | Regression_sin_phi: 0.5231 | LR: 8.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5144s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3921.5 | uss_MB: 3138.1 | child_rss_MB: 4005.6 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:49,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:29,480] INFO : Step 13700/50000 rank0 | Train Loss: 9.3738 | Train Loss Unweighted: 3.0819 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7559 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1367 | Regression_eta: 0.0116 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0230 | Classification_binary: 2.7559 | Regression_cos_phi: 0.6355 | Regression_energy: 1.8910 | Regression_eta: 0.5697 | Regression_pt: 2.0428 | Regression_sin_phi: 0.4559 | LR: 8.43e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.7337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1051 | max_loss: 0.0033 | max_backward: 0.3926 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3892 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3921.5 | uss_MB: 3140.2 | child_rss_MB: 4009.0 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 12:51:29,696] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:10,375] INFO : Step 13800/50000 rank0 | Train Loss: 9.2593 | Train Loss Unweighted: 2.9632 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6360 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1385 | Regression_eta: 0.0105 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9374 | Classification_binary: 2.6360 | Regression_cos_phi: 0.6473 | Regression_energy: 1.9157 | Regression_eta: 0.5141 | Regression_pt: 2.0523 | Regression_sin_phi: 0.5565 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3080 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.3881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3920.3 | uss_MB: 3137.4 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:52:10,590] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:50,474] INFO : Step 13900/50000 rank0 | Train Loss: 9.1070 | Train Loss Unweighted: 3.0335 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7213 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1326 | Regression_eta: 0.0111 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9435 | Classification_binary: 2.7213 | Regression_cos_phi: 0.5938 | Regression_energy: 1.8330 | Regression_eta: 0.5450 | Regression_pt: 1.9175 | Regression_sin_phi: 0.5529 | LR: 8.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5197s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2230 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3928 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3920.3 | uss_MB: 3137.6 | child_rss_MB: 4012.1 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:50,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:30,217] INFO : Step 14000/50000 rank0 | Train Loss: 9.1316 | Train Loss Unweighted: 2.9412 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.6177 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1398 | Regression_eta: 0.0131 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7860 | Classification_binary: 2.6177 | Regression_cos_phi: 0.6367 | Regression_energy: 1.9336 | Regression_eta: 0.6418 | Regression_pt: 2.0073 | Regression_sin_phi: 0.5085 | LR: 8.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0266 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3920.3 | uss_MB: 3137.7 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:53:30,440] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:30,459] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 12:53:30,632] INFO : Running validation on rank0 +rank0 [2026-10-28 12:53:30,632] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:53:30,632] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:53:31,964] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 12:53:32,107] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:53:32,237] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:53:32,309] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:53:32,377] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:53:32,445] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:53:32,515] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:53:32,583] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:53:32,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:53:32,825] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:53:32,955] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:53:33,025] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:53:33,094] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:53:33,162] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:53:33,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:53:33,300] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:53:33,407] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:53:33,414] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:53:33,417] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:53:33,420] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:53:33,421] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:53:33,424] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:53:33,427] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:53:33,430] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:53:33,431] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:53:33,434] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:53:33,436] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:53:33,438] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:53:33,440] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:53:33,442] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:53:33,445] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:53:33,446] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:53:41,021] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.1316 | Valid Loss=9.2182 | Train Loss Unweighted=2.9412 | Valid Loss Unweighted=2.9859 | Stale=0 | ETA=260.0m +rank0 [2026-10-28 12:53:41,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:41,094] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:53:41,097] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:41,252] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:43,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:43,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:44,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:45,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:45,426] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:53:45,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:45,485] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:53:45,488] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:45,577] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:47,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:48,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:48,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:49,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:49,587] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:53:49,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:49,645] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:53:49,648] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:53:49,753] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:51,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:52,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:52,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:53,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:53,677] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:53,685] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:53,937] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:54,383] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:55,803] INFO : Plotted jets +rank0 [2026-10-28 12:53:56,881] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:56,914] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:57,135] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:57,507] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:59,377] INFO : Plotted jets +rank0 [2026-10-28 12:54:00,492] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:00,524] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:00,724] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:01,136] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:02,512] INFO : Plotted jets +rank0 [2026-10-28 12:54:03,611] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:43,612] INFO : Step 14100/50000 rank0 | Train Loss: 9.3844 | Train Loss Unweighted: 2.9438 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6077 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1424 | Regression_eta: 0.0108 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9504 | Classification_binary: 2.6077 | Regression_cos_phi: 0.5913 | Regression_energy: 1.9694 | Regression_eta: 0.5310 | Regression_pt: 2.1121 | Regression_sin_phi: 0.6227 | LR: 8.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5405s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0729 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2032 | max_optimizer_step: 0.1568 | max_tensorboard_logging: 0.2769 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4123.5 | uss_MB: 3342.0 | child_rss_MB: 4008.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:54:43,838] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:23,517] INFO : Step 14200/50000 rank0 | Train Loss: 9.4260 | Train Loss Unweighted: 3.0011 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6655 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1452 | Regression_eta: 0.0128 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8619 | Classification_binary: 2.6655 | Regression_cos_phi: 0.5978 | Regression_energy: 2.0083 | Regression_eta: 0.6269 | Regression_pt: 2.0711 | Regression_sin_phi: 0.5945 | LR: 8.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0428 | max_backward: 0.2035 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3683 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3997.6 | uss_MB: 3215.4 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:55:23,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:03,651] INFO : Step 14300/50000 rank0 | Train Loss: 9.5671 | Train Loss Unweighted: 2.9844 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6255 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1556 | Regression_eta: 0.0109 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9523 | Classification_binary: 2.6255 | Regression_cos_phi: 0.5396 | Regression_energy: 2.1518 | Regression_eta: 0.5369 | Regression_pt: 2.3111 | Regression_sin_phi: 0.4498 | LR: 8.30e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5144s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0461 | max_backward: 0.2031 | max_optimizer_step: 0.1181 | max_tensorboard_logging: 0.3630 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3997.6 | uss_MB: 3215.6 | child_rss_MB: 4015.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:56:03,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:43,797] INFO : Step 14400/50000 rank0 | Train Loss: 9.2978 | Train Loss Unweighted: 2.9826 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6399 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1488 | Regression_eta: 0.0103 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9530 | Classification_binary: 2.6399 | Regression_cos_phi: 0.5056 | Regression_energy: 2.0573 | Regression_eta: 0.5061 | Regression_pt: 2.1885 | Regression_sin_phi: 0.4475 | LR: 8.28e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5807s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.1006 | max_backward: 0.2032 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3692 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3997.6 | uss_MB: 3215.6 | child_rss_MB: 4013.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:56:44,009] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:23,900] INFO : Step 14500/50000 rank0 | Train Loss: 9.0609 | Train Loss Unweighted: 2.9192 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1386 | Regression_eta: 0.0091 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9417 | Classification_binary: 2.5905 | Regression_cos_phi: 0.5683 | Regression_energy: 1.9166 | Regression_eta: 0.4452 | Regression_pt: 2.1346 | Regression_sin_phi: 0.4640 | LR: 8.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4953s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0330 | max_backward: 0.2033 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3560 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3215.7 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:57:24,113] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:58:04,280] INFO : Step 14600/50000 rank0 | Train Loss: 9.3270 | Train Loss Unweighted: 3.0363 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7036 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1416 | Regression_eta: 0.0097 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9244 | Classification_binary: 2.7036 | Regression_cos_phi: 0.5812 | Regression_energy: 1.9579 | Regression_eta: 0.4772 | Regression_pt: 2.1174 | Regression_sin_phi: 0.5653 | LR: 8.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1575 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 3997.6 | uss_MB: 3215.8 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:58:04,505] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:44,881] INFO : Step 14700/50000 rank0 | Train Loss: 9.1428 | Train Loss Unweighted: 2.9849 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6524 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1435 | Regression_eta: 0.0111 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8871 | Classification_binary: 2.6524 | Regression_cos_phi: 0.5322 | Regression_energy: 1.9849 | Regression_eta: 0.5437 | Regression_pt: 2.1297 | Regression_sin_phi: 0.4129 | LR: 8.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.2701 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3215.8 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:58:45,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:25,000] INFO : Step 14800/50000 rank0 | Train Loss: 9.1982 | Train Loss Unweighted: 3.0344 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7157 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1341 | Regression_eta: 0.0110 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.7157 | Regression_cos_phi: 0.6078 | Regression_energy: 1.8550 | Regression_eta: 0.5383 | Regression_pt: 1.9922 | Regression_sin_phi: 0.5530 | LR: 8.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4734s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0778 | max_loss: 0.0242 | max_backward: 0.2031 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 3997.7 | uss_MB: 3215.6 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:59:25,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:05,106] INFO : Step 14900/50000 rank0 | Train Loss: 9.6396 | Train Loss Unweighted: 3.0810 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7351 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1478 | Regression_eta: 0.0104 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9001 | Classification_binary: 2.7351 | Regression_cos_phi: 0.6337 | Regression_energy: 2.0438 | Regression_eta: 0.5089 | Regression_pt: 2.1932 | Regression_sin_phi: 0.6247 | LR: 8.17e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4651s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0198 | max_backward: 0.2032 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.3512 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3997.7 | uss_MB: 3215.6 | child_rss_MB: 4010.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:00:05,320] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:44,760] INFO : Step 15000/50000 rank0 | Train Loss: 8.1627 | Train Loss Unweighted: 2.7987 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5168 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1191 | Regression_eta: 0.0100 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8974 | Classification_binary: 2.5168 | Regression_cos_phi: 0.4549 | Regression_energy: 1.6473 | Regression_eta: 0.4930 | Regression_pt: 1.7738 | Regression_sin_phi: 0.3796 | LR: 8.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5088s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0946 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3997.7 | uss_MB: 3215.8 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:00:44,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:45,003] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 13:00:45,175] INFO : Running validation on rank0 +rank0 [2026-10-28 13:00:45,175] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:00:45,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:00:46,465] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 13:00:46,608] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:00:46,736] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:00:46,809] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:00:46,877] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:00:46,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:00:47,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:00:47,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:00:47,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:00:47,325] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:00:47,455] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:00:47,525] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:00:47,594] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:00:47,663] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:00:47,731] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:00:47,800] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:00:47,911] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:00:47,918] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:00:47,921] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:00:47,924] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:00:47,927] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:00:47,927] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:00:47,930] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:00:47,931] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:00:47,934] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:00:47,935] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:00:47,938] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:00:47,938] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:00:47,941] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:00:47,944] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:00:47,947] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:00:47,950] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:00:55,529] INFO : VALIDATION | Step=15000/50000 | Train Loss=8.1627 | Valid Loss=9.1146 | Train Loss Unweighted=2.7987 | Valid Loss Unweighted=2.9645 | Stale=0 | ETA=252.9m +rank0 [2026-10-28 13:00:55,540] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:55,602] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:00:55,607] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:00:55,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:57,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:58,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:58,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:59,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:59,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:59,575] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:59,636] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:00:59,639] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:59,726] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:01,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:02,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:02,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:03,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:03,640] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:03,643] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:03,696] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:01:03,699] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:01:03,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:05,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:06,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:06,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:07,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:07,756] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:01:07,764] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:08,015] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:08,461] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:10,344] INFO : Plotted jets +rank0 [2026-10-28 13:01:11,437] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:11,470] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:11,695] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:12,066] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:13,439] INFO : Plotted jets +rank0 [2026-10-28 13:01:14,551] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:14,585] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:14,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:15,199] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:16,581] INFO : Plotted jets +rank0 [2026-10-28 13:01:17,669] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:57,890] INFO : Step 15100/50000 rank0 | Train Loss: 8.8114 | Train Loss Unweighted: 2.9253 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6249 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1246 | Regression_eta: 0.0088 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9486 | Classification_binary: 2.6249 | Regression_cos_phi: 0.6174 | Regression_energy: 1.7226 | Regression_eta: 0.4330 | Regression_pt: 1.8771 | Regression_sin_phi: 0.5877 | LR: 8.12e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.6338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3743 | max_tensorboard_flush: 0.0054 | Memory: rss_MB: 4210.2 | uss_MB: 3430.0 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:01:58,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:37,390] INFO : Step 15200/50000 rank0 | Train Loss: 8.9394 | Train Loss Unweighted: 2.9543 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6455 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1305 | Regression_eta: 0.0099 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9633 | Classification_binary: 2.6455 | Regression_cos_phi: 0.6335 | Regression_energy: 1.8048 | Regression_eta: 0.4843 | Regression_pt: 1.9167 | Regression_sin_phi: 0.4913 | LR: 8.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3096 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.3654 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4068.4 | uss_MB: 3287.8 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:02:37,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:16,876] INFO : Step 15300/50000 rank0 | Train Loss: 9.9386 | Train Loss Unweighted: 3.0629 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6988 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1560 | Regression_eta: 0.0116 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9494 | Classification_binary: 2.6988 | Regression_cos_phi: 0.6470 | Regression_energy: 2.1571 | Regression_eta: 0.5702 | Regression_pt: 2.3101 | Regression_sin_phi: 0.6060 | LR: 8.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2948 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 4068.4 | uss_MB: 3288.0 | child_rss_MB: 4004.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:03:17,091] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:57,155] INFO : Step 15400/50000 rank0 | Train Loss: 9.1750 | Train Loss Unweighted: 2.9511 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6208 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1409 | Regression_eta: 0.0104 | Regression_pt: 0.1396 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9112 | Classification_binary: 2.6208 | Regression_cos_phi: 0.5142 | Regression_energy: 1.9485 | Regression_eta: 0.5128 | Regression_pt: 2.0996 | Regression_sin_phi: 0.5679 | LR: 8.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3207 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3485 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4069.4 | uss_MB: 3290.5 | child_rss_MB: 4003.8 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 92.9 +rank0 [2026-10-28 13:03:57,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:36,848] INFO : Step 15500/50000 rank0 | Train Loss: 8.7099 | Train Loss Unweighted: 2.8486 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.5340 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1358 | Regression_eta: 0.0094 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9659 | Classification_binary: 2.5340 | Regression_cos_phi: 0.5230 | Regression_energy: 1.8780 | Regression_eta: 0.4626 | Regression_pt: 1.9894 | Regression_sin_phi: 0.3570 | LR: 8.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4604s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0263 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2026 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4069.4 | uss_MB: 3288.5 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:04:37,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:16,814] INFO : Step 15600/50000 rank0 | Train Loss: 9.2667 | Train Loss Unweighted: 2.9987 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6570 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1489 | Regression_eta: 0.0090 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 2.6570 | Regression_cos_phi: 0.5468 | Regression_energy: 2.0593 | Regression_eta: 0.4407 | Regression_pt: 2.2013 | Regression_sin_phi: 0.5041 | LR: 8.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1831 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2760 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4069.4 | uss_MB: 3288.6 | child_rss_MB: 4006.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:05:17,030] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:56,261] INFO : Step 15700/50000 rank0 | Train Loss: 9.1317 | Train Loss Unweighted: 2.9941 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6754 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1349 | Regression_eta: 0.0112 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9136 | Classification_binary: 2.6754 | Regression_cos_phi: 0.5757 | Regression_energy: 1.8653 | Regression_eta: 0.5525 | Regression_pt: 1.9898 | Regression_sin_phi: 0.5594 | LR: 7.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4231s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1091 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.2745 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4069.4 | uss_MB: 3288.7 | child_rss_MB: 4004.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:05:56,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:36,516] INFO : Step 15800/50000 rank0 | Train Loss: 8.9628 | Train Loss Unweighted: 2.8917 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5645 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1402 | Regression_eta: 0.0077 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9176 | Classification_binary: 2.5645 | Regression_cos_phi: 0.5680 | Regression_energy: 1.9384 | Regression_eta: 0.3797 | Regression_pt: 2.1140 | Regression_sin_phi: 0.4807 | LR: 7.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4516s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0464 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1662 | max_tensorboard_logging: 0.2654 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4069.4 | uss_MB: 3287.7 | child_rss_MB: 4007.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:36,742] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:16,922] INFO : Step 15900/50000 rank0 | Train Loss: 9.5369 | Train Loss Unweighted: 3.0345 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6952 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1429 | Regression_eta: 0.0117 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9073 | Classification_binary: 2.6952 | Regression_cos_phi: 0.6641 | Regression_energy: 1.9763 | Regression_eta: 0.5766 | Regression_pt: 2.1530 | Regression_sin_phi: 0.5643 | LR: 7.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5706s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3077 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3666 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4069.4 | uss_MB: 3288.0 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:17,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:56,960] INFO : Step 16000/50000 rank0 | Train Loss: 9.2916 | Train Loss Unweighted: 2.9937 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.6671 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1377 | Regression_eta: 0.0116 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.6671 | Regression_cos_phi: 0.6967 | Regression_energy: 1.9036 | Regression_eta: 0.5678 | Regression_pt: 2.0544 | Regression_sin_phi: 0.4857 | LR: 7.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3028 | max_optimizer_step: 0.1182 | max_tensorboard_logging: 0.3594 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4069.4 | uss_MB: 3288.3 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:57,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:57,191] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 13:07:57,466] INFO : Running validation on rank0 +rank0 [2026-10-28 13:07:57,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:07:57,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:07:58,839] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 13:07:58,982] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:07:59,112] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:07:59,184] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:07:59,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:07:59,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:07:59,388] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:07:59,457] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:07:59,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:07:59,699] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:07:59,828] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:07:59,898] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:07:59,967] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:08:00,035] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:08:00,105] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:08:00,173] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:08:00,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:08:00,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:08:00,299] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:08:00,302] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:08:00,303] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:08:00,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:08:00,309] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:08:00,312] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:08:00,313] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:08:00,315] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:08:00,316] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:08:00,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:08:00,320] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:08:00,322] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:08:00,323] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:08:00,326] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:08:07,659] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.2916 | Valid Loss=9.1660 | Train Loss Unweighted=2.9937 | Valid Loss Unweighted=2.9781 | Stale=1 | ETA=245.6m +rank0 [2026-10-28 13:08:07,668] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:07,766] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:08:07,770] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:07,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:09,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:10,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:11,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:12,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:12,311] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:08:12,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:12,371] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:08:12,374] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:12,468] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:14,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:14,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:15,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:16,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:16,221] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:16,224] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:16,284] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:08:16,286] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:08:16,375] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:18,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:18,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:19,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:20,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:20,309] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:20,317] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:20,567] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:21,015] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:22,959] INFO : Plotted jets +rank0 [2026-10-28 13:08:24,054] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:24,087] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:24,311] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:24,680] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:26,051] INFO : Plotted jets +rank0 [2026-10-28 13:08:27,162] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:27,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:27,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:27,810] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:29,195] INFO : Plotted jets +rank0 [2026-10-28 13:08:30,286] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:10,600] INFO : Step 16100/50000 rank0 | Train Loss: 8.9912 | Train Loss Unweighted: 2.9049 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5937 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1329 | Regression_eta: 0.0109 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8786 | Classification_binary: 2.5937 | Regression_cos_phi: 0.6501 | Regression_energy: 1.8372 | Regression_eta: 0.5335 | Regression_pt: 1.9007 | Regression_sin_phi: 0.5974 | LR: 7.89e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.2709 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4296.1 | uss_MB: 3515.9 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:09:10,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:50,248] INFO : Step 16200/50000 rank0 | Train Loss: 9.5920 | Train Loss Unweighted: 3.0162 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6767 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1412 | Regression_eta: 0.0125 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9709 | Classification_binary: 2.6767 | Regression_cos_phi: 0.6829 | Regression_energy: 1.9523 | Regression_eta: 0.6153 | Regression_pt: 2.1489 | Regression_sin_phi: 0.5450 | LR: 7.86e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5402s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0697 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4142.3 | uss_MB: 3360.8 | child_rss_MB: 4002.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:09:50,475] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:30,140] INFO : Step 16300/50000 rank0 | Train Loss: 8.2695 | Train Loss Unweighted: 2.7899 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5026 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1219 | Regression_eta: 0.0099 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8643 | Classification_binary: 2.5026 | Regression_cos_phi: 0.4970 | Regression_energy: 1.6850 | Regression_eta: 0.4877 | Regression_pt: 1.8037 | Regression_sin_phi: 0.4291 | LR: 7.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.1001 | max_backward: 0.2029 | max_optimizer_step: 0.1220 | max_tensorboard_logging: 0.3389 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4142.3 | uss_MB: 3361.1 | child_rss_MB: 4001.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:10:30,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:10,273] INFO : Step 16400/50000 rank0 | Train Loss: 9.1104 | Train Loss Unweighted: 2.9067 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.5844 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1364 | Regression_eta: 0.0127 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 2.5844 | Regression_cos_phi: 0.5962 | Regression_energy: 1.8867 | Regression_eta: 0.6249 | Regression_pt: 2.0038 | Regression_sin_phi: 0.4231 | LR: 7.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5206s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0559 | max_backward: 0.2034 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3467 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4142.3 | uss_MB: 3361.1 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:11:10,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:50,405] INFO : Step 16500/50000 rank0 | Train Loss: 9.1897 | Train Loss Unweighted: 3.0006 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6727 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1388 | Regression_eta: 0.0105 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8532 | Classification_binary: 2.6727 | Regression_cos_phi: 0.6446 | Regression_energy: 1.9200 | Regression_eta: 0.5141 | Regression_pt: 2.1089 | Regression_sin_phi: 0.4760 | LR: 7.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4768s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1855 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.2666 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4142.3 | uss_MB: 3361.2 | child_rss_MB: 4002.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:11:50,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:30,440] INFO : Step 16600/50000 rank0 | Train Loss: 8.6120 | Train Loss Unweighted: 2.8724 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.5689 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1279 | Regression_eta: 0.0073 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9398 | Classification_binary: 2.5689 | Regression_cos_phi: 0.5767 | Regression_energy: 1.7693 | Regression_eta: 0.3591 | Regression_pt: 1.9439 | Regression_sin_phi: 0.4543 | LR: 7.77e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5063s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2215 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3639 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4142.3 | uss_MB: 3361.2 | child_rss_MB: 4001.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:12:30,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:10,996] INFO : Step 16700/50000 rank0 | Train Loss: 8.8424 | Train Loss Unweighted: 2.8430 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5357 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1281 | Regression_eta: 0.0101 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9071 | Classification_binary: 2.5357 | Regression_cos_phi: 0.6199 | Regression_energy: 1.7715 | Regression_eta: 0.4979 | Regression_pt: 1.9230 | Regression_sin_phi: 0.5872 | LR: 7.74e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1437 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4142.3 | uss_MB: 3361.3 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:13:11,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:51,411] INFO : Step 16800/50000 rank0 | Train Loss: 9.4870 | Train Loss Unweighted: 2.9813 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6361 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1477 | Regression_eta: 0.0104 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9234 | Classification_binary: 2.6361 | Regression_cos_phi: 0.6189 | Regression_energy: 2.0426 | Regression_eta: 0.5092 | Regression_pt: 2.1972 | Regression_sin_phi: 0.5596 | LR: 7.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4580s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1656 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.2720 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4142.5 | uss_MB: 3360.9 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:13:51,628] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:31,639] INFO : Step 16900/50000 rank0 | Train Loss: 9.1282 | Train Loss Unweighted: 2.9636 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6367 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1397 | Regression_eta: 0.0096 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8731 | Classification_binary: 2.6367 | Regression_cos_phi: 0.6440 | Regression_energy: 1.9318 | Regression_eta: 0.4739 | Regression_pt: 2.0833 | Regression_sin_phi: 0.4854 | LR: 7.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5318s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0659 | max_backward: 0.2030 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.3613 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4142.5 | uss_MB: 3361.0 | child_rss_MB: 4006.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:14:31,855] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:11,173] INFO : Step 17000/50000 rank0 | Train Loss: 9.5655 | Train Loss Unweighted: 2.9205 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5604 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1534 | Regression_eta: 0.0112 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.5604 | Regression_cos_phi: 0.6133 | Regression_energy: 2.1208 | Regression_eta: 0.5502 | Regression_pt: 2.3645 | Regression_sin_phi: 0.5049 | LR: 7.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1000 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.2715 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4142.5 | uss_MB: 3361.3 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:15:11,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:11,407] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 13:15:11,583] INFO : Running validation on rank0 +rank0 [2026-10-28 13:15:11,583] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:15:11,583] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:15:12,871] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 13:15:13,014] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:15:13,142] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:15:13,215] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:15:13,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:15:13,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:15:13,421] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:15:13,488] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:15:13,602] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:15:13,732] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:15:13,861] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:15:13,932] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:15:14,001] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:15:14,069] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:15:14,137] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:15:14,204] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:15:14,322] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:15:14,329] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:15:14,332] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:15:14,333] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:15:14,335] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:15:14,338] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:15:14,339] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:15:14,342] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:15:14,345] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:15:14,348] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:15:14,349] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:15:14,351] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:15:14,354] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:15:14,354] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:15:14,357] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:15:14,360] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:15:21,641] INFO : VALIDATION | Step=17000/50000 | Train Loss=9.5655 | Valid Loss=9.0179 | Train Loss Unweighted=2.9205 | Valid Loss Unweighted=2.9269 | Stale=0 | ETA=238.4m +rank0 [2026-10-28 13:15:21,652] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:21,732] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:15:21,736] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:21,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:23,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:24,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:25,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:25,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:25,981] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:25,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:26,049] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:15:26,052] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:26,134] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:27,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:28,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:29,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:30,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:30,144] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:30,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:30,201] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:15:30,204] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:15:30,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:32,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:32,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:33,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:34,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:34,252] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:34,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:34,511] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:34,957] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:36,375] INFO : Plotted jets +rank0 [2026-10-28 13:15:37,972] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:38,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:38,231] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:38,599] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:39,970] INFO : Plotted jets +rank0 [2026-10-28 13:15:41,079] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:41,111] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:41,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:41,732] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:43,119] INFO : Plotted jets +rank0 [2026-10-28 13:15:44,214] INFO : Plotted jet ratio +rank0 [2026-10-28 13:16:24,999] INFO : Step 17100/50000 rank0 | Train Loss: 8.8490 | Train Loss Unweighted: 3.0253 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.7086 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1377 | Regression_eta: 0.0088 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8794 | Classification_binary: 2.7086 | Regression_cos_phi: 0.4695 | Regression_energy: 1.9047 | Regression_eta: 0.4345 | Regression_pt: 2.0249 | Regression_sin_phi: 0.4275 | LR: 7.64e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6784s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1294 | max_loss: 0.0034 | max_backward: 0.3884 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.2828 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4352.4 | uss_MB: 3574.2 | child_rss_MB: 3998.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:16:25,214] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:04,228] INFO : Step 17200/50000 rank0 | Train Loss: 9.0451 | Train Loss Unweighted: 2.9714 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6594 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1312 | Regression_eta: 0.0122 | Regression_pt: 0.1290 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8969 | Classification_binary: 2.6594 | Regression_cos_phi: 0.6266 | Regression_energy: 1.8137 | Regression_eta: 0.5999 | Regression_pt: 1.9402 | Regression_sin_phi: 0.5085 | LR: 7.62e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1094 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1212 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4215.2 | uss_MB: 3435.7 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:17:04,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:44,164] INFO : Step 17300/50000 rank0 | Train Loss: 9.1159 | Train Loss Unweighted: 3.0300 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7018 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1408 | Regression_eta: 0.0111 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9115 | Classification_binary: 2.7018 | Regression_cos_phi: 0.4737 | Regression_energy: 1.9466 | Regression_eta: 0.5473 | Regression_pt: 2.1046 | Regression_sin_phi: 0.4305 | LR: 7.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3084 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4215.2 | uss_MB: 3435.8 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:17:44,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:23,944] INFO : Step 17400/50000 rank0 | Train Loss: 8.9442 | Train Loss Unweighted: 2.9607 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6371 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1405 | Regression_eta: 0.0087 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8079 | Classification_binary: 2.6371 | Regression_cos_phi: 0.5797 | Regression_energy: 1.9432 | Regression_eta: 0.4272 | Regression_pt: 2.0788 | Regression_sin_phi: 0.4703 | LR: 7.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5094s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0419 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.1613 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4215.2 | uss_MB: 3435.9 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:18:24,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:03,992] INFO : Step 17500/50000 rank0 | Train Loss: 8.2494 | Train Loss Unweighted: 2.6931 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.3967 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1260 | Regression_eta: 0.0083 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8651 | Classification_binary: 2.3967 | Regression_cos_phi: 0.5367 | Regression_energy: 1.7420 | Regression_eta: 0.4068 | Regression_pt: 1.9035 | Regression_sin_phi: 0.3986 | LR: 7.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4447s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0937 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.1212 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4215.2 | uss_MB: 3435.9 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:19:04,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:44,089] INFO : Step 17600/50000 rank0 | Train Loss: 9.0382 | Train Loss Unweighted: 2.8387 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5315 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1289 | Regression_eta: 0.0114 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8783 | Classification_binary: 2.5315 | Regression_cos_phi: 0.7775 | Regression_energy: 1.7831 | Regression_eta: 0.5614 | Regression_pt: 1.8434 | Regression_sin_phi: 0.6629 | LR: 7.52e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0420 | max_backward: 0.2027 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3588 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4215.2 | uss_MB: 3436.0 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:19:44,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:24,194] INFO : Step 17700/50000 rank0 | Train Loss: 8.5508 | Train Loss Unweighted: 2.9042 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6025 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1280 | Regression_eta: 0.0080 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9100 | Classification_binary: 2.6025 | Regression_cos_phi: 0.4957 | Regression_energy: 1.7703 | Regression_eta: 0.3912 | Regression_pt: 1.9365 | Regression_sin_phi: 0.4446 | LR: 7.49e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8385s | Phase Times: max_pre_step_barrier: 0.1029 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0035 | max_backward: 0.3891 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.3826 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4216.2 | uss_MB: 3438.4 | child_rss_MB: 4000.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:20:24,406] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:04,815] INFO : Step 17800/50000 rank0 | Train Loss: 8.3597 | Train Loss Unweighted: 2.7624 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4722 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1244 | Regression_eta: 0.0133 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8288 | Classification_binary: 2.4722 | Regression_cos_phi: 0.4952 | Regression_energy: 1.7202 | Regression_eta: 0.6532 | Regression_pt: 1.7753 | Regression_sin_phi: 0.4148 | LR: 7.47e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5011s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1816 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2839 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4215.4 | uss_MB: 3435.8 | child_rss_MB: 3997.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:21:05,034] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:45,462] INFO : Step 17900/50000 rank0 | Train Loss: 8.4624 | Train Loss Unweighted: 2.8109 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5032 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1330 | Regression_eta: 0.0071 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8499 | Classification_binary: 2.5032 | Regression_cos_phi: 0.4821 | Regression_energy: 1.8388 | Regression_eta: 0.3466 | Regression_pt: 1.9887 | Regression_sin_phi: 0.4531 | LR: 7.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5510s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2501 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4215.4 | uss_MB: 3435.9 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:21:45,679] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,653] INFO : Step 18000/50000 rank0 | Train Loss: 8.3531 | Train Loss Unweighted: 2.7323 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4349 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1263 | Regression_eta: 0.0083 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8068 | Classification_binary: 2.4349 | Regression_cos_phi: 0.5853 | Regression_energy: 1.7465 | Regression_eta: 0.4069 | Regression_pt: 1.9044 | Regression_sin_phi: 0.4683 | LR: 7.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5753s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0886 | max_backward: 0.2032 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.3618 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4215.4 | uss_MB: 3436.0 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:22:25,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,914] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 13:22:26,095] INFO : Running validation on rank0 +rank0 [2026-10-28 13:22:26,095] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:22:26,095] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:22:27,394] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 13:22:27,539] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:22:27,669] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:22:27,741] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:22:27,810] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:22:27,877] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:22:27,945] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:22:28,013] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:22:28,127] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:22:28,258] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:22:28,388] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:22:28,458] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:22:28,527] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:22:28,594] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:22:28,664] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:22:28,733] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:22:28,831] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:22:28,837] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:22:28,840] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:22:28,841] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:22:28,844] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:22:28,845] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:22:28,848] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:22:28,848] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:22:28,851] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:22:28,854] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:22:28,855] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:22:28,858] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:22:28,861] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:22:28,863] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:22:28,864] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:22:28,867] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:22:36,303] INFO : VALIDATION | Step=18000/50000 | Train Loss=8.3531 | Valid Loss=8.9329 | Train Loss Unweighted=2.7323 | Valid Loss Unweighted=2.9060 | Stale=0 | ETA=231.2m +rank0 [2026-10-28 13:22:36,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:36,375] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:22:36,378] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:36,593] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:38,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:39,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:39,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:40,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:40,687] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:22:40,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:40,762] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:22:40,765] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:40,850] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:42,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:43,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:43,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:44,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:44,694] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:44,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:44,774] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:22:44,777] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:22:44,848] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:46,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:47,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:48,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:48,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:48,821] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:22:48,828] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:49,080] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:49,526] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:50,947] INFO : Plotted jets +rank0 [2026-10-28 13:22:52,536] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:52,568] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:52,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:53,166] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:54,534] INFO : Plotted jets +rank0 [2026-10-28 13:22:55,646] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:55,678] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 13:22:55,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 13:22:56,296] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:57,671] INFO : Plotted jets +rank0 [2026-10-28 13:22:58,767] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:39,452] INFO : Step 18100/50000 rank0 | Train Loss: 9.4806 | Train Loss Unweighted: 2.9017 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5625 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1432 | Regression_eta: 0.0145 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8596 | Classification_binary: 2.5625 | Regression_cos_phi: 0.7076 | Regression_energy: 1.9805 | Regression_eta: 0.7129 | Regression_pt: 2.1192 | Regression_sin_phi: 0.5382 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5871s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1302 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4410.5 | uss_MB: 3630.6 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:23:39,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:19,416] INFO : Step 18200/50000 rank0 | Train Loss: 8.7523 | Train Loss Unweighted: 2.9918 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6805 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1349 | Regression_eta: 0.0089 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8484 | Classification_binary: 2.6805 | Regression_cos_phi: 0.5074 | Regression_energy: 1.8649 | Regression_eta: 0.4373 | Regression_pt: 1.9904 | Regression_sin_phi: 0.4233 | LR: 7.36e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5562s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3032 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4286.3 | uss_MB: 3507.4 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:24:19,632] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:59,233] INFO : Step 18300/50000 rank0 | Train Loss: 8.8529 | Train Loss Unweighted: 2.8591 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.5424 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1338 | Regression_eta: 0.0110 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8743 | Classification_binary: 2.5424 | Regression_cos_phi: 0.5607 | Regression_energy: 1.8508 | Regression_eta: 0.5426 | Regression_pt: 2.0245 | Regression_sin_phi: 0.4575 | LR: 7.34e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4286.3 | uss_MB: 3508.6 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:24:59,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:38,992] INFO : Step 18400/50000 rank0 | Train Loss: 8.4651 | Train Loss Unweighted: 2.8007 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4982 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1287 | Regression_eta: 0.0082 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 2.4982 | Regression_cos_phi: 0.5352 | Regression_energy: 1.7792 | Regression_eta: 0.4023 | Regression_pt: 1.9460 | Regression_sin_phi: 0.4455 | LR: 7.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1458 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2249 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4286.3 | uss_MB: 3508.6 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:25:39,219] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:18,759] INFO : Step 18500/50000 rank0 | Train Loss: 8.8058 | Train Loss Unweighted: 2.8294 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5058 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1406 | Regression_eta: 0.0089 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.5058 | Regression_cos_phi: 0.5391 | Regression_energy: 1.9449 | Regression_eta: 0.4376 | Regression_pt: 2.0705 | Regression_sin_phi: 0.4576 | LR: 7.29e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5733s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2608 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4286.3 | uss_MB: 3508.7 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:26:18,972] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:59,520] INFO : Step 18600/50000 rank0 | Train Loss: 9.6734 | Train Loss Unweighted: 2.9094 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.5557 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1510 | Regression_eta: 0.0108 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9224 | Classification_binary: 2.5557 | Regression_cos_phi: 0.7724 | Regression_energy: 2.0885 | Regression_eta: 0.5314 | Regression_pt: 2.2291 | Regression_sin_phi: 0.5738 | LR: 7.26e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6526s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3104 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3941 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4286.3 | uss_MB: 3508.8 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:26:59,733] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:27:39,022] INFO : Step 18700/50000 rank0 | Train Loss: 8.9254 | Train Loss Unweighted: 2.9527 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6460 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1291 | Regression_eta: 0.0112 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8242 | Classification_binary: 2.6460 | Regression_cos_phi: 0.7092 | Regression_energy: 1.7855 | Regression_eta: 0.5524 | Regression_pt: 1.9167 | Regression_sin_phi: 0.4916 | LR: 7.24e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6288s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3072 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4286.3 | uss_MB: 3508.8 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:27:39,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:19,436] INFO : Step 18800/50000 rank0 | Train Loss: 8.9246 | Train Loss Unweighted: 2.8704 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5517 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1358 | Regression_eta: 0.0098 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8860 | Classification_binary: 2.5517 | Regression_cos_phi: 0.6399 | Regression_energy: 1.8784 | Regression_eta: 0.4827 | Regression_pt: 2.0180 | Regression_sin_phi: 0.4680 | LR: 7.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5589s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1158 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4285.0 | uss_MB: 3507.4 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:28:19,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:59,368] INFO : Step 18900/50000 rank0 | Train Loss: 9.1478 | Train Loss Unweighted: 2.9490 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6083 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1479 | Regression_eta: 0.0079 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9106 | Classification_binary: 2.6083 | Regression_cos_phi: 0.5515 | Regression_energy: 2.0450 | Regression_eta: 0.3858 | Regression_pt: 2.2187 | Regression_sin_phi: 0.4279 | LR: 7.18e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1781 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.2977 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4285.0 | uss_MB: 3508.6 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:28:59,583] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:40,008] INFO : Step 19000/50000 rank0 | Train Loss: 8.3544 | Train Loss Unweighted: 2.7908 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4964 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1251 | Regression_eta: 0.0091 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7991 | Classification_binary: 2.4964 | Regression_cos_phi: 0.5866 | Regression_energy: 1.7299 | Regression_eta: 0.4479 | Regression_pt: 1.8810 | Regression_sin_phi: 0.4134 | LR: 7.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6320s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3114 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3843 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4285.0 | uss_MB: 3508.7 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:29:40,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:40,337] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 13:29:40,515] INFO : Running validation on rank0 +rank0 [2026-10-28 13:29:40,515] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:29:40,515] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:29:41,811] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 13:29:41,954] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:29:42,082] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:29:42,155] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:29:42,223] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:29:42,291] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:29:42,360] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:29:42,428] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:29:42,543] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:29:42,673] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:29:42,803] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:29:42,874] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:29:42,943] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:29:43,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:29:43,079] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:29:43,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:29:43,247] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:29:43,253] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:29:43,256] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:29:43,257] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:29:43,260] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:29:43,263] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:29:43,266] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:29:43,267] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:29:43,270] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:29:43,273] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:29:43,275] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:29:43,276] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:29:43,279] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:29:43,282] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:29:43,284] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:29:43,285] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:29:50,636] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.3544 | Valid Loss=8.8895 | Train Loss Unweighted=2.7908 | Valid Loss Unweighted=2.8965 | Stale=0 | ETA=224.0m +rank0 [2026-10-28 13:29:50,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:50,756] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:29:50,759] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:29:50,922] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:29:52,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:29:53,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:29:54,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:29:55,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:29:55,128] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:29:55,131] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:55,189] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:29:55,192] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:29:55,288] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:29:57,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:29:57,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:29:58,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:29:59,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:29:59,240] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:29:59,243] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:59,302] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:29:59,305] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:29:59,400] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:30:01,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:30:01,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:30:02,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:30:03,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:30:03,370] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:30:03,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:30:03,643] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:30:04,090] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:05,505] INFO : Plotted jets +rank0 [2026-10-28 13:30:06,604] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:06,636] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:30:06,858] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:30:07,737] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:09,104] INFO : Plotted jets +rank0 [2026-10-28 13:30:10,213] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:10,247] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 13:30:10,465] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 13:30:10,876] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:12,252] INFO : Plotted jets +rank0 [2026-10-28 13:30:13,344] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:53,856] INFO : Step 19100/50000 rank0 | Train Loss: 8.9014 | Train Loss Unweighted: 2.9570 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6472 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1308 | Regression_eta: 0.0114 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.6472 | Regression_cos_phi: 0.6696 | Regression_energy: 1.8093 | Regression_eta: 0.5586 | Regression_pt: 1.9562 | Regression_sin_phi: 0.4015 | LR: 7.13e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.1290 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.2723 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4476.7 | uss_MB: 3701.8 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:30:54,081] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:31:33,954] INFO : Step 19200/50000 rank0 | Train Loss: 9.1562 | Train Loss Unweighted: 2.8450 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5221 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1348 | Regression_eta: 0.0126 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9288 | Classification_binary: 2.5221 | Regression_cos_phi: 0.6249 | Regression_energy: 1.8636 | Regression_eta: 0.6214 | Regression_pt: 2.0106 | Regression_sin_phi: 0.5847 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5058s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0309 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2027 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4357.1 | uss_MB: 3581.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:31:34,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:14,453] INFO : Step 19300/50000 rank0 | Train Loss: 8.7028 | Train Loss Unweighted: 2.7954 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4835 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1315 | Regression_eta: 0.0101 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7829 | Classification_binary: 2.4835 | Regression_cos_phi: 0.6052 | Regression_energy: 1.8183 | Regression_eta: 0.4959 | Regression_pt: 2.0108 | Regression_sin_phi: 0.5061 | LR: 7.08e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5605s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.2189 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4357.1 | uss_MB: 3582.6 | child_rss_MB: 3992.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:14,673] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:54,659] INFO : Step 19400/50000 rank0 | Train Loss: 8.1865 | Train Loss Unweighted: 2.8038 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5172 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1227 | Regression_eta: 0.0086 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8191 | Classification_binary: 2.5172 | Regression_cos_phi: 0.4854 | Regression_energy: 1.6970 | Regression_eta: 0.4226 | Regression_pt: 1.8203 | Regression_sin_phi: 0.4248 | LR: 7.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3108 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:54,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:34,861] INFO : Step 19500/50000 rank0 | Train Loss: 9.9376 | Train Loss Unweighted: 2.9532 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5764 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1631 | Regression_eta: 0.0103 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8831 | Classification_binary: 2.5764 | Regression_cos_phi: 0.7547 | Regression_energy: 2.2552 | Regression_eta: 0.5084 | Regression_pt: 2.4339 | Regression_sin_phi: 0.5259 | LR: 7.02e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1802 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3990.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:33:35,075] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:34:14,737] INFO : Step 19600/50000 rank0 | Train Loss: 8.8084 | Train Loss Unweighted: 2.8028 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4812 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1387 | Regression_eta: 0.0093 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8578 | Classification_binary: 2.4812 | Regression_cos_phi: 0.5619 | Regression_energy: 1.9174 | Regression_eta: 0.4593 | Regression_pt: 2.0513 | Regression_sin_phi: 0.4797 | LR: 7.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6506s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.3053 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4357.1 | uss_MB: 3582.9 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:14,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:34:55,425] INFO : Step 19700/50000 rank0 | Train Loss: 8.4243 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4064 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1219 | Regression_eta: 0.0107 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8412 | Classification_binary: 2.4064 | Regression_cos_phi: 0.6379 | Regression_energy: 1.6860 | Regression_eta: 0.5280 | Regression_pt: 1.8354 | Regression_sin_phi: 0.4894 | LR: 6.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1848 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.2919 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:55,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:35:35,274] INFO : Step 19800/50000 rank0 | Train Loss: 8.5778 | Train Loss Unweighted: 2.6571 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3471 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1314 | Regression_eta: 0.0111 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8492 | Classification_binary: 2.3471 | Regression_cos_phi: 0.5786 | Regression_energy: 1.8178 | Regression_eta: 0.5460 | Regression_pt: 1.9564 | Regression_sin_phi: 0.4827 | LR: 6.94e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3100 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.4001 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4356.0 | uss_MB: 3581.2 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:35:35,487] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:15,056] INFO : Step 19900/50000 rank0 | Train Loss: 9.2925 | Train Loss Unweighted: 2.8592 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5260 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1403 | Regression_eta: 0.0102 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8466 | Classification_binary: 2.5260 | Regression_cos_phi: 0.6866 | Regression_energy: 1.9402 | Regression_eta: 0.5031 | Regression_pt: 2.1098 | Regression_sin_phi: 0.6802 | LR: 6.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5665s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1028 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4357.0 | uss_MB: 3582.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:36:15,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:54,856] INFO : Step 20000/50000 rank0 | Train Loss: 8.6915 | Train Loss Unweighted: 2.9342 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1303 | Regression_eta: 0.0098 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9110 | Classification_binary: 2.6305 | Regression_cos_phi: 0.5273 | Regression_energy: 1.8015 | Regression_eta: 0.4812 | Regression_pt: 1.8987 | Regression_sin_phi: 0.4413 | LR: 6.89e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.7457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1047 | max_loss: 0.0034 | max_backward: 0.3928 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4357.0 | uss_MB: 3585.1 | child_rss_MB: 3994.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:36:55,071] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:55,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 13:36:55,328] INFO : Running validation on rank0 +rank0 [2026-10-28 13:36:55,328] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:36:55,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:36:56,753] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 13:36:56,896] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:36:57,026] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:36:57,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:36:57,167] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:36:57,235] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:36:57,303] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:36:57,372] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:36:57,484] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:36:57,614] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:36:57,743] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:36:57,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:36:57,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:36:57,948] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:36:58,017] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:36:58,086] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:36:58,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:36:58,210] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:36:58,212] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:36:58,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:36:58,216] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:36:58,219] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:36:58,220] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:36:58,222] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:36:58,223] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:36:58,226] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:36:58,229] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:36:58,231] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:36:58,234] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:36:58,236] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:36:58,238] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:36:58,241] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:37:05,689] INFO : VALIDATION | Step=20000/50000 | Train Loss=8.6915 | Valid Loss=8.7360 | Train Loss Unweighted=2.9342 | Valid Loss Unweighted=2.8415 | Stale=0 | ETA=216.8m +rank0 [2026-10-28 13:37:05,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:05,772] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:37:05,775] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:05,936] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:07,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:08,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:09,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:09,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:09,980] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:37:09,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:10,087] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:37:10,090] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:10,258] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:12,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:12,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:13,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:14,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:14,168] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:37:14,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:14,243] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:37:14,246] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:37:14,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:16,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:16,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:17,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:18,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:18,177] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:37:18,185] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:18,437] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:18,883] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:20,302] INFO : Plotted jets +rank0 [2026-10-28 13:37:21,912] INFO : Plotted jet ratio +rank0 [2026-10-28 13:37:21,945] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:22,169] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:22,543] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:23,914] INFO : Plotted jets +rank0 [2026-10-28 13:37:25,024] INFO : Plotted jet ratio +rank0 [2026-10-28 13:37:25,055] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 13:37:25,258] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 13:37:25,671] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:27,045] INFO : Plotted jets +rank0 [2026-10-28 13:37:28,133] INFO : Plotted jet ratio +rank0 [2026-10-28 13:38:08,072] INFO : Step 20100/50000 rank0 | Train Loss: 8.6005 | Train Loss Unweighted: 2.8225 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5255 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1248 | Regression_eta: 0.0112 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9079 | Classification_binary: 2.5255 | Regression_cos_phi: 0.5549 | Regression_energy: 1.7252 | Regression_eta: 0.5490 | Regression_pt: 1.8418 | Regression_sin_phi: 0.4962 | LR: 6.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6229s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1364 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4565.3 | uss_MB: 3791.9 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:38:08,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:48,547] INFO : Step 20200/50000 rank0 | Train Loss: 8.4530 | Train Loss Unweighted: 2.8603 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5777 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1186 | Regression_eta: 0.0103 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7888 | Classification_binary: 2.5777 | Regression_cos_phi: 0.6280 | Regression_energy: 1.6399 | Regression_eta: 0.5070 | Regression_pt: 1.7360 | Regression_sin_phi: 0.5756 | LR: 6.84e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6168s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.1040 | max_backward: 0.2037 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4430.6 | uss_MB: 3655.1 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:38:48,762] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:39:28,657] INFO : Step 20300/50000 rank0 | Train Loss: 8.7180 | Train Loss Unweighted: 2.8173 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5043 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1331 | Regression_eta: 0.0089 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.5043 | Regression_cos_phi: 0.5367 | Regression_energy: 1.8405 | Regression_eta: 0.4382 | Regression_pt: 1.9995 | Regression_sin_phi: 0.5314 | LR: 6.81e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1857 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:39:28,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:08,604] INFO : Step 20400/50000 rank0 | Train Loss: 8.6127 | Train Loss Unweighted: 2.7832 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4907 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1220 | Regression_eta: 0.0134 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8037 | Classification_binary: 2.4907 | Regression_cos_phi: 0.6269 | Regression_energy: 1.6873 | Regression_eta: 0.6593 | Regression_pt: 1.7870 | Regression_sin_phi: 0.5578 | LR: 6.78e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3105 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3924 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 13:40:08,819] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:49,925] INFO : Step 20500/50000 rank0 | Train Loss: 8.2046 | Train Loss Unweighted: 2.7946 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5021 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1247 | Regression_eta: 0.0086 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7926 | Classification_binary: 2.5021 | Regression_cos_phi: 0.4341 | Regression_energy: 1.7240 | Regression_eta: 0.4217 | Regression_pt: 1.9006 | Regression_sin_phi: 0.4297 | LR: 6.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3769 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:40:50,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:41:30,416] INFO : Step 20600/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.7819 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4715 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1332 | Regression_eta: 0.0089 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8619 | Classification_binary: 2.4715 | Regression_cos_phi: 0.5378 | Regression_energy: 1.8417 | Regression_eta: 0.4356 | Regression_pt: 1.9911 | Regression_sin_phi: 0.4206 | LR: 6.73e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2126 | max_loss: 0.0035 | max_backward: 0.2874 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4431.6 | uss_MB: 3659.1 | child_rss_MB: 3993.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:41:30,643] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:11,175] INFO : Step 20700/50000 rank0 | Train Loss: 9.3085 | Train Loss Unweighted: 2.8955 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5585 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1420 | Regression_eta: 0.0108 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8939 | Classification_binary: 2.5585 | Regression_cos_phi: 0.6888 | Regression_energy: 1.9638 | Regression_eta: 0.5295 | Regression_pt: 2.1598 | Regression_sin_phi: 0.5142 | LR: 6.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3116 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4431.6 | uss_MB: 3657.1 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:11,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:51,820] INFO : Step 20800/50000 rank0 | Train Loss: 9.4649 | Train Loss Unweighted: 2.9083 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5562 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1514 | Regression_eta: 0.0115 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9422 | Classification_binary: 2.5562 | Regression_cos_phi: 0.5871 | Regression_energy: 2.0941 | Regression_eta: 0.5645 | Regression_pt: 2.2511 | Regression_sin_phi: 0.4696 | LR: 6.67e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3090 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3940 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4431.1 | uss_MB: 3655.0 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:52,033] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:43:32,012] INFO : Step 20900/50000 rank0 | Train Loss: 8.7815 | Train Loss Unweighted: 2.7980 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1329 | Regression_eta: 0.0109 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8653 | Classification_binary: 2.4868 | Regression_cos_phi: 0.5314 | Regression_energy: 1.8383 | Regression_eta: 0.5345 | Regression_pt: 1.9316 | Regression_sin_phi: 0.5936 | LR: 6.65e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5397s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1787 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4433.1 | uss_MB: 3656.4 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:43:32,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:11,928] INFO : Step 21000/50000 rank0 | Train Loss: 8.7870 | Train Loss Unweighted: 2.9215 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6127 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1299 | Regression_eta: 0.0101 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8406 | Classification_binary: 2.6127 | Regression_cos_phi: 0.5266 | Regression_energy: 1.7959 | Regression_eta: 0.4972 | Regression_pt: 1.9782 | Regression_sin_phi: 0.5358 | LR: 6.62e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6279s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3086 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4433.1 | uss_MB: 3656.7 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:44:12,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:12,257] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 13:44:12,433] INFO : Running validation on rank0 +rank0 [2026-10-28 13:44:12,433] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:44:12,433] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:44:13,723] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 13:44:13,866] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:44:13,997] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:44:14,070] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:44:14,138] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:44:14,206] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:44:14,276] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:44:14,345] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:44:14,458] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:44:14,588] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:44:14,719] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:44:14,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:44:14,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:44:14,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:44:14,995] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:44:15,064] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:44:15,160] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:44:15,166] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:44:15,169] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:44:15,170] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:44:15,172] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:44:15,173] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:44:15,176] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:44:15,177] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:44:15,180] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:44:15,182] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:44:15,183] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:44:15,186] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:44:15,187] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:44:15,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:44:15,192] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:44:15,195] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:44:22,632] INFO : VALIDATION | Step=21000/50000 | Train Loss=8.7870 | Valid Loss=8.7762 | Train Loss Unweighted=2.9215 | Valid Loss Unweighted=2.8587 | Stale=1 | ETA=209.7m +rank0 [2026-10-28 13:44:22,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:22,725] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:44:22,728] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:22,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:24,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:25,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:26,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:26,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:26,973] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:44:26,976] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:27,032] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:44:27,035] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:27,130] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:29,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:29,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:30,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:30,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:31,050] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:44:31,054] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:31,109] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:44:31,112] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:44:31,205] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:32,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:33,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:34,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:35,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:35,199] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:44:35,207] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:35,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:35,904] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:37,325] INFO : Plotted jets +rank0 [2026-10-28 13:44:38,954] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:38,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:39,213] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:39,589] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:40,964] INFO : Plotted jets +rank0 [2026-10-28 13:44:42,076] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:42,108] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 13:44:42,314] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 13:44:42,726] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:44,111] INFO : Plotted jets +rank0 [2026-10-28 13:44:45,202] INFO : Plotted jet ratio +rank0 [2026-10-28 13:45:25,044] INFO : Step 21100/50000 rank0 | Train Loss: 9.1603 | Train Loss Unweighted: 2.9861 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6502 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1451 | Regression_eta: 0.0080 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.6502 | Regression_cos_phi: 0.5427 | Regression_energy: 2.0070 | Regression_eta: 0.3926 | Regression_pt: 2.1796 | Regression_sin_phi: 0.5291 | LR: 6.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4806s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0099 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1540 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4647.8 | uss_MB: 3871.0 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:45:25,273] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:45:28,646] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 13:45:28,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:46:04,784] INFO : Step 21200/50000 rank0 | Train Loss: 8.6887 | Train Loss Unweighted: 2.8540 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5406 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1346 | Regression_eta: 0.0089 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8919 | Classification_binary: 2.5406 | Regression_cos_phi: 0.5094 | Regression_energy: 1.8607 | Regression_eta: 0.4376 | Regression_pt: 2.0043 | Regression_sin_phi: 0.4442 | LR: 6.56e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5521s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0124 | max_backward: 0.2026 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3553 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4504.8 | uss_MB: 3725.6 | child_rss_MB: 3989.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 13:46:04,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:44,984] INFO : Step 21300/50000 rank0 | Train Loss: 8.9032 | Train Loss Unweighted: 2.8642 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5416 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1399 | Regression_eta: 0.0101 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9045 | Classification_binary: 2.5416 | Regression_cos_phi: 0.4843 | Regression_energy: 1.9340 | Regression_eta: 0.4975 | Regression_pt: 2.0265 | Regression_sin_phi: 0.5149 | LR: 6.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5908s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4505.8 | uss_MB: 3727.3 | child_rss_MB: 3989.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:46:45,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:47:24,692] INFO : Step 21400/50000 rank0 | Train Loss: 8.8688 | Train Loss Unweighted: 2.7766 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4588 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1337 | Regression_eta: 0.0118 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8862 | Classification_binary: 2.4588 | Regression_cos_phi: 0.5479 | Regression_energy: 1.8489 | Regression_eta: 0.5782 | Regression_pt: 2.0087 | Regression_sin_phi: 0.5402 | LR: 6.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6506s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2797 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4505.8 | uss_MB: 3727.5 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:47:24,905] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:04,093] INFO : Step 21500/50000 rank0 | Train Loss: 8.5595 | Train Loss Unweighted: 2.8174 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5183 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1274 | Regression_eta: 0.0105 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8405 | Classification_binary: 2.5183 | Regression_cos_phi: 0.5454 | Regression_energy: 1.7619 | Regression_eta: 0.5166 | Regression_pt: 1.8647 | Regression_sin_phi: 0.5121 | LR: 6.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5571s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0586 | max_backward: 0.2031 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3442 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4505.8 | uss_MB: 3727.7 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:48:04,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:43,303] INFO : Step 21600/50000 rank0 | Train Loss: 8.8485 | Train Loss Unweighted: 2.7954 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4770 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1346 | Regression_eta: 0.0090 | Regression_pt: 0.1355 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 2.4770 | Regression_cos_phi: 0.6364 | Regression_energy: 1.8608 | Regression_eta: 0.4430 | Regression_pt: 2.0383 | Regression_sin_phi: 0.5356 | LR: 6.45e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5861s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2532 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 4505.8 | uss_MB: 3727.8 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:48:43,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:49:23,335] INFO : Step 21700/50000 rank0 | Train Loss: 8.2277 | Train Loss Unweighted: 2.7442 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1309 | Regression_eta: 0.0082 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7969 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4774 | Regression_energy: 1.8100 | Regression_eta: 0.4050 | Regression_pt: 1.9270 | Regression_sin_phi: 0.3669 | LR: 6.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2595 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4505.8 | uss_MB: 3727.8 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:49:23,548] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:03,504] INFO : Step 21800/50000 rank0 | Train Loss: 8.2401 | Train Loss Unweighted: 2.7430 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4539 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1220 | Regression_eta: 0.0080 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7828 | Classification_binary: 2.4539 | Regression_cos_phi: 0.5258 | Regression_energy: 1.6873 | Regression_eta: 0.3944 | Regression_pt: 1.8496 | Regression_sin_phi: 0.5464 | LR: 6.40e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6512s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3088 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4505.2 | uss_MB: 3725.7 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:50:03,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:43,587] INFO : Step 21900/50000 rank0 | Train Loss: 8.4682 | Train Loss Unweighted: 2.8525 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5576 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1265 | Regression_eta: 0.0103 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8414 | Classification_binary: 2.5576 | Regression_cos_phi: 0.4868 | Regression_energy: 1.7487 | Regression_eta: 0.5044 | Regression_pt: 1.8383 | Regression_sin_phi: 0.4909 | LR: 6.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5877s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.2617 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4506.2 | uss_MB: 3727.2 | child_rss_MB: 3992.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:50:43,801] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:24,156] INFO : Step 22000/50000 rank0 | Train Loss: 9.2807 | Train Loss Unweighted: 2.8615 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5113 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1525 | Regression_eta: 0.0099 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9062 | Classification_binary: 2.5113 | Regression_cos_phi: 0.5237 | Regression_energy: 2.1090 | Regression_eta: 0.4875 | Regression_pt: 2.2517 | Regression_sin_phi: 0.4914 | LR: 6.34e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6164s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3044 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4506.2 | uss_MB: 3727.4 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:51:24,368] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:24,386] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 13:51:24,671] INFO : Running validation on rank0 +rank0 [2026-10-28 13:51:24,671] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:51:24,671] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:51:25,865] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 13:51:26,010] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:51:26,140] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:51:26,214] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:51:26,282] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:51:26,351] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:51:26,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:51:26,488] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:51:26,600] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:51:26,730] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:51:26,861] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:51:26,932] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:51:27,000] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:51:27,068] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:51:27,137] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:51:27,205] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:51:27,316] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:51:27,409] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:51:27,412] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:51:27,414] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:51:27,417] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:51:27,418] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:51:27,421] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:51:27,424] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:51:27,425] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:51:27,427] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:51:27,428] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:51:27,431] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:51:27,434] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:51:27,437] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:51:27,437] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:51:27,440] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:51:34,881] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.2807 | Valid Loss=8.6936 | Train Loss Unweighted=2.8615 | Valid Loss Unweighted=2.8315 | Stale=0 | ETA=202.4m +rank0 [2026-10-28 13:51:34,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:34,952] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:51:34,955] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:35,032] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:37,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:37,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:38,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:39,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:39,168] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:39,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:39,227] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:51:39,230] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:39,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:41,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:41,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:42,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:43,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:43,317] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:43,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:43,375] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:51:43,378] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:51:43,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:45,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:45,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:46,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:47,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:47,460] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:47,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:47,717] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:48,173] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:49,588] INFO : Plotted jets +rank0 [2026-10-28 13:51:51,187] INFO : Plotted jet ratio +rank0 [2026-10-28 13:51:51,219] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:51,440] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:51,812] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:53,189] INFO : Plotted jets +rank0 [2026-10-28 13:51:54,300] INFO : Plotted jet ratio +rank0 [2026-10-28 13:51:54,333] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 13:51:54,537] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 13:51:54,947] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:56,329] INFO : Plotted jets +rank0 [2026-10-28 13:51:57,416] INFO : Plotted jet ratio +rank0 [2026-10-28 13:52:37,740] INFO : Step 22100/50000 rank0 | Train Loss: 7.6891 | Train Loss Unweighted: 2.6576 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3845 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1172 | Regression_eta: 0.0077 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7739 | Classification_binary: 2.3845 | Regression_cos_phi: 0.4335 | Regression_energy: 1.6213 | Regression_eta: 0.3801 | Regression_pt: 1.7640 | Regression_sin_phi: 0.3317 | LR: 6.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6150s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1368 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4712.8 | uss_MB: 3935.8 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:52:37,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:17,798] INFO : Step 22200/50000 rank0 | Train Loss: 8.0657 | Train Loss Unweighted: 2.6653 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3825 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1190 | Regression_eta: 0.0093 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8310 | Classification_binary: 2.3825 | Regression_cos_phi: 0.4882 | Regression_energy: 1.6456 | Regression_eta: 0.4580 | Regression_pt: 1.7917 | Regression_sin_phi: 0.4687 | LR: 6.29e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1063 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3114 | max_optimizer_step: 0.1305 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4712.8 | uss_MB: 3935.8 | child_rss_MB: 4016.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:53:18,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:57,684] INFO : Step 22300/50000 rank0 | Train Loss: 8.9171 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5284 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1419 | Regression_eta: 0.0093 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.5284 | Regression_cos_phi: 0.4968 | Regression_energy: 1.9618 | Regression_eta: 0.4559 | Regression_pt: 2.1379 | Regression_sin_phi: 0.4849 | LR: 6.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6013s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1277 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.2222 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4582.3 | uss_MB: 3806.6 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:53:57,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:38,036] INFO : Step 22400/50000 rank0 | Train Loss: 8.3987 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5368 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1292 | Regression_eta: 0.0076 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 2.5368 | Regression_cos_phi: 0.5283 | Regression_energy: 1.7865 | Regression_eta: 0.3747 | Regression_pt: 1.8937 | Regression_sin_phi: 0.4508 | LR: 6.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6479s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3046 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4582.3 | uss_MB: 3807.0 | child_rss_MB: 3992.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:54:38,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:18,039] INFO : Step 22500/50000 rank0 | Train Loss: 7.9802 | Train Loss Unweighted: 2.8337 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5668 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1130 | Regression_eta: 0.0084 | Regression_pt: 0.1106 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8215 | Classification_binary: 2.5668 | Regression_cos_phi: 0.5451 | Regression_energy: 1.5633 | Regression_eta: 0.4110 | Regression_pt: 1.6641 | Regression_sin_phi: 0.4085 | LR: 6.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6357s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0031 | max_backward: 0.3038 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3747 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4582.3 | uss_MB: 3807.2 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:55:18,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:57,947] INFO : Step 22600/50000 rank0 | Train Loss: 8.2434 | Train Loss Unweighted: 2.7290 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4346 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1262 | Regression_eta: 0.0091 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8245 | Classification_binary: 2.4346 | Regression_cos_phi: 0.4661 | Regression_energy: 1.7459 | Regression_eta: 0.4494 | Regression_pt: 1.8718 | Regression_sin_phi: 0.4512 | LR: 6.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5957s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.2477 | max_optimizer_step: 0.1288 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4582.3 | uss_MB: 3807.2 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:55:58,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:56:38,034] INFO : Step 22700/50000 rank0 | Train Loss: 8.7923 | Train Loss Unweighted: 2.8562 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5564 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1238 | Regression_eta: 0.0105 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8508 | Classification_binary: 2.5564 | Regression_cos_phi: 0.6522 | Regression_energy: 1.7123 | Regression_eta: 0.5148 | Regression_pt: 1.8679 | Regression_sin_phi: 0.6378 | LR: 6.15e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1833 | max_loss: 0.0034 | max_backward: 0.2036 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4582.3 | uss_MB: 3807.3 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:56:38,248] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:18,120] INFO : Step 22800/50000 rank0 | Train Loss: 8.1902 | Train Loss Unweighted: 2.7412 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4551 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1205 | Regression_eta: 0.0079 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8718 | Classification_binary: 2.4551 | Regression_cos_phi: 0.5672 | Regression_energy: 1.6660 | Regression_eta: 0.3884 | Regression_pt: 1.8211 | Regression_sin_phi: 0.4206 | LR: 6.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6259s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1857 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4582.3 | uss_MB: 3807.4 | child_rss_MB: 3990.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:57:18,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:57,776] INFO : Step 22900/50000 rank0 | Train Loss: 8.5141 | Train Loss Unweighted: 2.7533 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4376 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1377 | Regression_eta: 0.0100 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.4376 | Regression_cos_phi: 0.5244 | Regression_energy: 1.9043 | Regression_eta: 0.4892 | Regression_pt: 2.0402 | Regression_sin_phi: 0.3413 | LR: 6.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5540s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0516 | max_backward: 0.2035 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3405 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4581.7 | uss_MB: 3806.6 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:57:57,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:37,723] INFO : Step 23000/50000 rank0 | Train Loss: 8.0713 | Train Loss Unweighted: 2.6961 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1154 | Regression_eta: 0.0101 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8141 | Classification_binary: 2.4237 | Regression_cos_phi: 0.6011 | Regression_energy: 1.5958 | Regression_eta: 0.4957 | Regression_pt: 1.6533 | Regression_sin_phi: 0.4875 | LR: 6.06e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0034 | max_backward: 0.2271 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4581.7 | uss_MB: 3807.1 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:58:37,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:38,047] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 13:58:38,223] INFO : Running validation on rank0 +rank0 [2026-10-28 13:58:38,223] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:58:38,223] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:58:39,522] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 13:58:39,665] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:58:39,795] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:58:39,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:58:39,936] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:58:40,004] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:58:40,073] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:58:40,142] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:58:40,256] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:58:40,385] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:58:40,515] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:58:40,586] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:58:40,654] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:58:40,721] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:58:40,789] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:58:40,858] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:58:40,954] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:58:40,960] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:58:40,962] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:58:40,964] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:58:40,965] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:58:40,967] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:58:40,968] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:58:40,971] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:58:40,974] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:58:40,977] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:58:40,978] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:58:40,981] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:58:40,983] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:58:40,984] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:58:40,986] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:58:40,989] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:58:48,307] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.0713 | Valid Loss=8.6660 | Train Loss Unweighted=2.6961 | Valid Loss Unweighted=2.8138 | Stale=0 | ETA=195.2m +rank0 [2026-10-28 13:58:48,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:48,377] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:58:48,381] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:58:48,456] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:50,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:50,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:51,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:58:52,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:58:52,229] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:58:52,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:52,288] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:58:52,290] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:58:52,389] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:54,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:54,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:55,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:58:56,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:58:56,285] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:58:56,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:56,342] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:58:56,344] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:58:56,451] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:58,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:58,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:59,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:59:00,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:59:00,415] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:59:00,423] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:59:00,679] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:59:01,126] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:02,548] INFO : Plotted jets +rank0 [2026-10-28 13:59:04,187] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:04,220] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:59:04,444] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:59:04,821] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:06,243] INFO : Plotted jets +rank0 [2026-10-28 13:59:07,337] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:07,369] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 13:59:07,569] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 13:59:07,982] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:09,360] INFO : Plotted jets +rank0 [2026-10-28 13:59:10,465] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:50,823] INFO : Step 23100/50000 rank0 | Train Loss: 8.7210 | Train Loss Unweighted: 2.8490 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5345 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1341 | Regression_eta: 0.0089 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7954 | Classification_binary: 2.5345 | Regression_cos_phi: 0.5529 | Regression_energy: 1.8547 | Regression_eta: 0.4365 | Regression_pt: 2.0352 | Regression_sin_phi: 0.5118 | LR: 6.04e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0450 | max_backward: 0.2026 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.3097 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4780.1 | uss_MB: 4007.1 | child_rss_MB: 3990.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:59:51,035] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:00:30,940] INFO : Step 23200/50000 rank0 | Train Loss: 8.1690 | Train Loss Unweighted: 2.7382 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1204 | Regression_eta: 0.0096 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8525 | Classification_binary: 2.4544 | Regression_cos_phi: 0.5437 | Regression_energy: 1.6648 | Regression_eta: 0.4734 | Regression_pt: 1.7778 | Regression_sin_phi: 0.4024 | LR: 6.01e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1860 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4780.1 | uss_MB: 4007.1 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:00:31,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:10,778] INFO : Step 23300/50000 rank0 | Train Loss: 8.2610 | Train Loss Unweighted: 2.7374 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4437 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1259 | Regression_eta: 0.0086 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8109 | Classification_binary: 2.4437 | Regression_cos_phi: 0.5753 | Regression_energy: 1.7404 | Regression_eta: 0.4237 | Regression_pt: 1.8694 | Regression_sin_phi: 0.3976 | LR: 5.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0386 | max_backward: 0.2031 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3325 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4651.5 | uss_MB: 3878.1 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:01:10,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:50,742] INFO : Step 23400/50000 rank0 | Train Loss: 8.4409 | Train Loss Unweighted: 2.6613 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3558 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1302 | Regression_eta: 0.0098 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8193 | Classification_binary: 2.3558 | Regression_cos_phi: 0.5466 | Regression_energy: 1.8005 | Regression_eta: 0.4833 | Regression_pt: 1.9422 | Regression_sin_phi: 0.4932 | LR: 5.95e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2487 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4651.5 | uss_MB: 3878.3 | child_rss_MB: 3993.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:01:50,953] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:02:30,823] INFO : Step 23500/50000 rank0 | Train Loss: 8.5490 | Train Loss Unweighted: 2.6953 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3944 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1266 | Regression_eta: 0.0134 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8170 | Classification_binary: 2.3944 | Regression_cos_phi: 0.5662 | Regression_energy: 1.7513 | Regression_eta: 0.6579 | Regression_pt: 1.8696 | Regression_sin_phi: 0.4925 | LR: 5.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5634s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1824 | max_loss: 0.0036 | max_backward: 0.2037 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4651.5 | uss_MB: 3878.4 | child_rss_MB: 3987.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:02:31,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:10,252] INFO : Step 23600/50000 rank0 | Train Loss: 8.8357 | Train Loss Unweighted: 2.9482 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6350 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1346 | Regression_eta: 0.0084 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7946 | Classification_binary: 2.6350 | Regression_cos_phi: 0.6046 | Regression_energy: 1.8609 | Regression_eta: 0.4145 | Regression_pt: 2.0000 | Regression_sin_phi: 0.5261 | LR: 5.90e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4651.5 | uss_MB: 3878.5 | child_rss_MB: 3988.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:03:10,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:49,941] INFO : Step 23700/50000 rank0 | Train Loss: 8.1938 | Train Loss Unweighted: 2.7824 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4951 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1235 | Regression_eta: 0.0086 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8722 | Classification_binary: 2.4951 | Regression_cos_phi: 0.4781 | Regression_energy: 1.7083 | Regression_eta: 0.4236 | Regression_pt: 1.8033 | Regression_sin_phi: 0.4133 | LR: 5.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7260s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.0034 | max_backward: 0.3810 | max_optimizer_step: 0.1270 | max_tensorboard_logging: 0.3539 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4652.5 | uss_MB: 3880.6 | child_rss_MB: 3990.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 14:03:50,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:04:30,460] INFO : Step 23800/50000 rank0 | Train Loss: 8.6763 | Train Loss Unweighted: 2.8378 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5286 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1327 | Regression_eta: 0.0087 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8504 | Classification_binary: 2.5286 | Regression_cos_phi: 0.5495 | Regression_energy: 1.8346 | Regression_eta: 0.4268 | Regression_pt: 1.9561 | Regression_sin_phi: 0.5303 | LR: 5.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5155s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0625 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.2761 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4652.5 | uss_MB: 3878.6 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:04:30,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:10,157] INFO : Step 23900/50000 rank0 | Train Loss: 8.9800 | Train Loss Unweighted: 2.9394 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6292 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1302 | Regression_eta: 0.0107 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9030 | Classification_binary: 2.6292 | Regression_cos_phi: 0.6787 | Regression_energy: 1.8001 | Regression_eta: 0.5279 | Regression_pt: 1.9368 | Regression_sin_phi: 0.5043 | LR: 5.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5496s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0574 | max_backward: 0.2034 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3414 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4653.0 | uss_MB: 3878.3 | child_rss_MB: 3989.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:05:10,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:50,193] INFO : Step 24000/50000 rank0 | Train Loss: 8.8756 | Train Loss Unweighted: 2.7751 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4565 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1352 | Regression_eta: 0.0103 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8351 | Classification_binary: 2.4565 | Regression_cos_phi: 0.7104 | Regression_energy: 1.8692 | Regression_eta: 0.5068 | Regression_pt: 2.0173 | Regression_sin_phi: 0.4803 | LR: 5.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1846 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4653.0 | uss_MB: 3878.5 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:05:50,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:50,428] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 14:05:50,600] INFO : Running validation on rank0 +rank0 [2026-10-28 14:05:50,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:05:50,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:05:51,891] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 14:05:52,036] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:05:52,165] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:05:52,238] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:05:52,307] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:05:52,375] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:05:52,444] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:05:52,512] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:05:52,625] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:05:52,755] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:05:52,891] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:05:52,961] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:05:53,030] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:05:53,099] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:05:53,167] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:05:53,236] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:05:53,335] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:05:53,342] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:05:53,345] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:05:53,348] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:05:53,349] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:05:53,351] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:05:53,354] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:05:53,355] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:05:53,358] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:05:53,361] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:05:53,364] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:05:53,366] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:05:53,368] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:05:53,370] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:05:53,371] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:05:53,374] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:06:00,801] INFO : VALIDATION | Step=24000/50000 | Train Loss=8.8756 | Valid Loss=8.6014 | Train Loss Unweighted=2.7751 | Valid Loss Unweighted=2.7943 | Stale=0 | ETA=187.9m +rank0 [2026-10-28 14:06:00,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:00,873] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:06:00,876] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:01,041] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:02,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:03,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:04,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:05,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:05,197] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:06:05,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:05,256] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:06:05,259] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:05,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:07,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:07,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:08,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:09,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:09,290] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:06:09,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:09,349] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:06:09,352] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:06:09,452] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:11,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:11,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:12,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:13,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:13,409] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:06:13,417] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:13,669] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:14,116] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:15,536] INFO : Plotted jets +rank0 [2026-10-28 14:06:17,202] INFO : Plotted jet ratio +rank0 [2026-10-28 14:06:17,234] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:17,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:17,830] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:19,210] INFO : Plotted jets +rank0 [2026-10-28 14:06:20,274] INFO : Plotted jet ratio +rank0 [2026-10-28 14:06:20,306] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 14:06:20,510] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 14:06:20,924] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:22,306] INFO : Plotted jets +rank0 [2026-10-28 14:06:23,419] INFO : Plotted jet ratio +rank0 [2026-10-28 14:07:04,154] INFO : Step 24100/50000 rank0 | Train Loss: 8.7307 | Train Loss Unweighted: 2.7990 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4931 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1296 | Regression_eta: 0.0104 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8923 | Classification_binary: 2.4931 | Regression_cos_phi: 0.6236 | Regression_energy: 1.7919 | Regression_eta: 0.5120 | Regression_pt: 1.8972 | Regression_sin_phi: 0.5206 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6853s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0883 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3094 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4855.8 | uss_MB: 4083.5 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:07:04,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:43,728] INFO : Step 24200/50000 rank0 | Train Loss: 8.0567 | Train Loss Unweighted: 2.7548 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4834 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1142 | Regression_eta: 0.0105 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8181 | Classification_binary: 2.4834 | Regression_cos_phi: 0.4659 | Regression_energy: 1.5792 | Regression_eta: 0.5147 | Regression_pt: 1.6710 | Regression_sin_phi: 0.5244 | LR: 5.73e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7035s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3067 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4855.8 | uss_MB: 4083.5 | child_rss_MB: 3986.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:07:43,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:08:23,336] INFO : Step 24300/50000 rank0 | Train Loss: 9.2852 | Train Loss Unweighted: 2.9776 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6438 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1437 | Regression_eta: 0.0115 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8828 | Classification_binary: 2.6438 | Regression_cos_phi: 0.6426 | Regression_energy: 1.9871 | Regression_eta: 0.5643 | Regression_pt: 2.1033 | Regression_sin_phi: 0.4613 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5554s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0722 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.2033 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4725.2 | uss_MB: 3954.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:08:23,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:03,726] INFO : Step 24400/50000 rank0 | Train Loss: 8.1835 | Train Loss Unweighted: 2.8827 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.6032 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1185 | Regression_eta: 0.0096 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7228 | Classification_binary: 2.6032 | Regression_cos_phi: 0.5069 | Regression_energy: 1.6389 | Regression_eta: 0.4740 | Regression_pt: 1.7838 | Regression_sin_phi: 0.4539 | LR: 5.67e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6779s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3087 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3821 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4726.2 | uss_MB: 3954.6 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:09:03,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:43,128] INFO : Step 24500/50000 rank0 | Train Loss: 8.0380 | Train Loss Unweighted: 2.6094 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.3305 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1182 | Regression_eta: 0.0097 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8908 | Classification_binary: 2.3305 | Regression_cos_phi: 0.5796 | Regression_energy: 1.6340 | Regression_eta: 0.4751 | Regression_pt: 1.7137 | Regression_sin_phi: 0.4144 | LR: 5.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5978s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2277 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4726.2 | uss_MB: 3954.6 | child_rss_MB: 3989.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:09:43,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:22,389] INFO : Step 24600/50000 rank0 | Train Loss: 8.9178 | Train Loss Unweighted: 2.7763 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4565 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1358 | Regression_eta: 0.0105 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8573 | Classification_binary: 2.4565 | Regression_cos_phi: 0.7352 | Regression_energy: 1.8779 | Regression_eta: 0.5148 | Regression_pt: 2.0162 | Regression_sin_phi: 0.4600 | LR: 5.61e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.6649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3080 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3794 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:10:22,602] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:11:02,736] INFO : Step 24700/50000 rank0 | Train Loss: 8.5762 | Train Loss Unweighted: 2.7821 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4810 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1263 | Regression_eta: 0.0090 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.4810 | Regression_cos_phi: 0.5978 | Regression_energy: 1.7459 | Regression_eta: 0.4423 | Regression_pt: 1.9101 | Regression_sin_phi: 0.5477 | LR: 5.58e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0847 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3989.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:11:02,959] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:11:42,230] INFO : Step 24800/50000 rank0 | Train Loss: 8.5547 | Train Loss Unweighted: 2.7416 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4371 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1283 | Regression_eta: 0.0127 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7849 | Classification_binary: 2.4371 | Regression_cos_phi: 0.5950 | Regression_energy: 1.7739 | Regression_eta: 0.6245 | Regression_pt: 1.9373 | Regression_sin_phi: 0.4019 | LR: 5.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:11:42,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:12:22,104] INFO : Step 24900/50000 rank0 | Train Loss: 7.9427 | Train Loss Unweighted: 2.7152 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4479 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1108 | Regression_eta: 0.0091 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8068 | Classification_binary: 2.4479 | Regression_cos_phi: 0.5387 | Regression_energy: 1.5323 | Regression_eta: 0.4457 | Regression_pt: 1.6750 | Regression_sin_phi: 0.4962 | LR: 5.53e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5266s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0247 | max_backward: 0.2034 | max_optimizer_step: 0.1271 | max_tensorboard_logging: 0.3350 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4724.1 | uss_MB: 3954.4 | child_rss_MB: 3989.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:12:22,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:02,068] INFO : Step 25000/50000 rank0 | Train Loss: 8.3151 | Train Loss Unweighted: 2.7759 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4929 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1193 | Regression_eta: 0.0100 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8282 | Classification_binary: 2.4929 | Regression_cos_phi: 0.6220 | Regression_energy: 1.6494 | Regression_eta: 0.4912 | Regression_pt: 1.7483 | Regression_sin_phi: 0.4831 | LR: 5.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6070s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0968 | max_backward: 0.2029 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4724.1 | uss_MB: 3954.6 | child_rss_MB: 3990.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:13:02,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:02,318] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 14:13:02,494] INFO : Running validation on rank0 +rank0 [2026-10-28 14:13:02,494] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:13:02,494] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:13:03,847] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 14:13:03,990] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:13:04,120] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:13:04,192] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:13:04,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:13:04,328] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:13:04,396] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:13:04,466] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:13:04,580] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:13:04,711] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:13:04,842] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:13:04,913] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:13:04,981] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:13:05,048] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:13:05,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:13:05,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:13:05,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:13:05,286] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:13:05,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:13:05,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:13:05,292] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:13:05,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:13:05,296] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:13:05,299] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:13:05,300] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:13:05,303] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:13:05,305] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:13:05,308] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:13:05,309] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:13:05,311] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:13:05,312] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:13:05,315] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:13:12,767] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.3151 | Valid Loss=8.5592 | Train Loss Unweighted=2.7759 | Valid Loss Unweighted=2.7818 | Stale=0 | ETA=180.7m +rank0 [2026-10-28 14:13:12,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:12,852] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:13:12,855] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:13,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:14,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:15,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:16,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:16,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:17,134] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:13:17,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:17,193] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:13:17,196] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:17,297] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:19,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:19,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:20,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:21,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:21,281] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:13:21,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:21,349] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:13:21,352] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:13:21,443] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:23,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:23,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:24,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:25,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:25,364] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:13:25,371] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:25,626] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:26,073] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:27,497] INFO : Plotted jets +rank0 [2026-10-28 14:13:28,596] INFO : Plotted jet ratio +rank0 [2026-10-28 14:13:28,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:28,852] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:29,232] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:31,171] INFO : Plotted jets +rank0 [2026-10-28 14:13:32,283] INFO : Plotted jet ratio +rank0 [2026-10-28 14:13:32,316] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 14:13:32,520] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 14:13:32,931] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:34,314] INFO : Plotted jets +rank0 [2026-10-28 14:13:35,416] INFO : Plotted jet ratio +rank0 [2026-10-28 14:14:15,559] INFO : Step 25100/50000 rank0 | Train Loss: 9.0638 | Train Loss Unweighted: 2.8492 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5147 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1437 | Regression_eta: 0.0096 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8861 | Classification_binary: 2.5147 | Regression_cos_phi: 0.5793 | Regression_energy: 1.9874 | Regression_eta: 0.4702 | Regression_pt: 2.1534 | Regression_sin_phi: 0.4727 | LR: 5.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6327s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2524 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4919.1 | uss_MB: 4149.4 | child_rss_MB: 3988.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:14:15,773] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:14:55,237] INFO : Step 25200/50000 rank0 | Train Loss: 8.3465 | Train Loss Unweighted: 2.8001 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5085 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1238 | Regression_eta: 0.0092 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8966 | Classification_binary: 2.5085 | Regression_cos_phi: 0.5010 | Regression_energy: 1.7126 | Regression_eta: 0.4505 | Regression_pt: 1.8341 | Regression_sin_phi: 0.4433 | LR: 5.44e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0654 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3088 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2812 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4919.1 | uss_MB: 4149.4 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:14:55,451] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:35,122] INFO : Step 25300/50000 rank0 | Train Loss: 8.2620 | Train Loss Unweighted: 2.7743 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4785 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1272 | Regression_eta: 0.0080 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8791 | Classification_binary: 2.4785 | Regression_cos_phi: 0.4830 | Regression_energy: 1.7590 | Regression_eta: 0.3944 | Regression_pt: 1.8921 | Regression_sin_phi: 0.3759 | LR: 5.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5930s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0933 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.2921 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4798.4 | uss_MB: 4028.6 | child_rss_MB: 3988.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:15:35,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:14,887] INFO : Step 25400/50000 rank0 | Train Loss: 8.6310 | Train Loss Unweighted: 2.7921 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4896 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1275 | Regression_eta: 0.0100 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8310 | Classification_binary: 2.4896 | Regression_cos_phi: 0.6011 | Regression_energy: 1.7629 | Regression_eta: 0.4937 | Regression_pt: 1.9043 | Regression_sin_phi: 0.5485 | LR: 5.39e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6994s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2824 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3879 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4798.4 | uss_MB: 4028.7 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:16:15,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:54,741] INFO : Step 25500/50000 rank0 | Train Loss: 8.1559 | Train Loss Unweighted: 2.7122 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4285 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1197 | Regression_eta: 0.0094 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8514 | Classification_binary: 2.4285 | Regression_cos_phi: 0.4926 | Regression_energy: 1.6554 | Regression_eta: 0.4597 | Regression_pt: 1.7805 | Regression_sin_phi: 0.4877 | LR: 5.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6721s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3076 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4798.4 | uss_MB: 4028.9 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:16:54,956] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:35,080] INFO : Step 25600/50000 rank0 | Train Loss: 8.3524 | Train Loss Unweighted: 2.7709 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4765 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1253 | Regression_eta: 0.0103 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8081 | Classification_binary: 2.4765 | Regression_cos_phi: 0.5107 | Regression_energy: 1.7326 | Regression_eta: 0.5071 | Regression_pt: 1.8644 | Regression_sin_phi: 0.4529 | LR: 5.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6607s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1836 | max_loss: 0.0037 | max_backward: 0.2029 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.2827 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4798.4 | uss_MB: 4028.9 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:17:35,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:18:15,795] INFO : Step 25700/50000 rank0 | Train Loss: 8.4961 | Train Loss Unweighted: 2.7574 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4381 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1382 | Regression_eta: 0.0084 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8782 | Classification_binary: 2.4381 | Regression_cos_phi: 0.3885 | Regression_energy: 1.9112 | Regression_eta: 0.4126 | Regression_pt: 2.0991 | Regression_sin_phi: 0.3684 | LR: 5.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1584 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.2906 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4798.4 | uss_MB: 4029.0 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:18:16,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:18:56,186] INFO : Step 25800/50000 rank0 | Train Loss: 8.3694 | Train Loss Unweighted: 2.6767 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3769 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1270 | Regression_eta: 0.0086 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8370 | Classification_binary: 2.3769 | Regression_cos_phi: 0.6006 | Regression_energy: 1.7558 | Regression_eta: 0.4241 | Regression_pt: 1.9133 | Regression_sin_phi: 0.4617 | LR: 5.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6485s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2954 | max_optimizer_step: 0.1263 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4798.4 | uss_MB: 4029.0 | child_rss_MB: 3987.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:18:56,397] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:19:36,114] INFO : Step 25900/50000 rank0 | Train Loss: 9.1298 | Train Loss Unweighted: 2.9160 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1389 | Regression_eta: 0.0101 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8386 | Classification_binary: 2.5905 | Regression_cos_phi: 0.6337 | Regression_energy: 1.9213 | Regression_eta: 0.4945 | Regression_pt: 2.0541 | Regression_sin_phi: 0.5971 | LR: 5.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6857s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3052 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4798.4 | uss_MB: 4028.7 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:19:36,326] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:16,142] INFO : Step 26000/50000 rank0 | Train Loss: 9.1757 | Train Loss Unweighted: 2.8125 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.4804 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1423 | Regression_eta: 0.0115 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9118 | Classification_binary: 2.4804 | Regression_cos_phi: 0.6900 | Regression_energy: 1.9674 | Regression_eta: 0.5647 | Regression_pt: 2.0709 | Regression_sin_phi: 0.4903 | LR: 5.22e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6658s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.3041 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4798.4 | uss_MB: 4028.8 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:20:16,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:16,373] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 14:20:16,646] INFO : Running validation on rank0 +rank0 [2026-10-28 14:20:16,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:20:16,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:20:17,931] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 14:20:18,076] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:20:18,208] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:20:18,281] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:20:18,349] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:20:18,415] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:20:18,484] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:20:18,553] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:20:18,666] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:20:18,798] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:20:18,929] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:20:19,000] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:20:19,068] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:20:19,137] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:20:19,205] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:20:19,274] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:20:19,393] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:20:19,400] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:20:19,403] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:20:19,406] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:20:19,407] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:20:19,409] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:20:19,411] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:20:19,413] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:20:19,416] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:20:19,417] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:20:19,419] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:20:19,420] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:20:19,423] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:20:19,425] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:20:19,428] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:20:19,429] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:20:26,804] INFO : VALIDATION | Step=26000/50000 | Train Loss=9.1757 | Valid Loss=8.5511 | Train Loss Unweighted=2.8125 | Valid Loss Unweighted=2.7827 | Stale=0 | ETA=173.4m +rank0 [2026-10-28 14:20:26,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:26,897] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:20:26,900] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:27,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:29,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:29,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:30,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:31,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:31,211] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:31,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:31,271] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:20:31,274] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:31,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:33,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:34,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:34,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:35,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:35,417] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:35,420] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:35,474] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:20:35,477] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:20:35,585] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:37,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:38,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:38,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:39,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:39,698] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:39,705] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:39,961] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:40,408] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:41,830] INFO : Plotted jets +rank0 [2026-10-28 14:20:43,475] INFO : Plotted jet ratio +rank0 [2026-10-28 14:20:43,507] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:43,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:44,111] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:45,497] INFO : Plotted jets +rank0 [2026-10-28 14:20:46,559] INFO : Plotted jet ratio +rank0 [2026-10-28 14:20:46,592] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 14:20:46,799] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 14:20:47,212] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:48,593] INFO : Plotted jets +rank0 [2026-10-28 14:20:49,695] INFO : Plotted jet ratio +rank0 [2026-10-28 14:21:30,200] INFO : Step 26100/50000 rank0 | Train Loss: 8.9560 | Train Loss Unweighted: 2.7926 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4672 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1399 | Regression_eta: 0.0089 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8697 | Classification_binary: 2.4672 | Regression_cos_phi: 0.6913 | Regression_energy: 1.9341 | Regression_eta: 0.4382 | Regression_pt: 2.0581 | Regression_sin_phi: 0.4973 | LR: 5.19e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6871s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3074 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3771 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5001.8 | uss_MB: 4231.7 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:21:30,414] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:10,826] INFO : Step 26200/50000 rank0 | Train Loss: 8.7222 | Train Loss Unweighted: 2.7342 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4306 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1269 | Regression_eta: 0.0115 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7769 | Classification_binary: 2.4306 | Regression_cos_phi: 0.7515 | Regression_energy: 1.7547 | Regression_eta: 0.5648 | Regression_pt: 1.8911 | Regression_sin_phi: 0.5525 | LR: 5.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6461s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0323 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3110 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5001.8 | uss_MB: 4231.8 | child_rss_MB: 4010.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:22:11,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:50,477] INFO : Step 26300/50000 rank0 | Train Loss: 8.0842 | Train Loss Unweighted: 2.6753 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3983 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1149 | Regression_eta: 0.0092 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8473 | Classification_binary: 2.3983 | Regression_cos_phi: 0.6224 | Regression_energy: 1.5885 | Regression_eta: 0.4516 | Regression_pt: 1.7421 | Regression_sin_phi: 0.4340 | LR: 5.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0548 | max_backward: 0.2037 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4872.2 | uss_MB: 4100.6 | child_rss_MB: 3985.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:22:50,690] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:23:30,295] INFO : Step 26400/50000 rank0 | Train Loss: 8.4589 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4531 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1298 | Regression_eta: 0.0091 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7989 | Classification_binary: 2.4531 | Regression_cos_phi: 0.5453 | Regression_energy: 1.7944 | Regression_eta: 0.4453 | Regression_pt: 1.9162 | Regression_sin_phi: 0.5056 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8219s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0206 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3953 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3956 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4872.2 | uss_MB: 4103.7 | child_rss_MB: 3986.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:23:30,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:10,963] INFO : Step 26500/50000 rank0 | Train Loss: 8.4298 | Train Loss Unweighted: 2.6826 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3734 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1339 | Regression_eta: 0.0100 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8166 | Classification_binary: 2.3734 | Regression_cos_phi: 0.4833 | Regression_energy: 1.8512 | Regression_eta: 0.4927 | Regression_pt: 1.9675 | Regression_sin_phi: 0.4451 | LR: 5.08e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3053 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4873.2 | uss_MB: 4101.9 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:24:11,177] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:50,909] INFO : Step 26600/50000 rank0 | Train Loss: 8.8323 | Train Loss Unweighted: 2.8044 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4920 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1322 | Regression_eta: 0.0128 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8143 | Classification_binary: 2.4920 | Regression_cos_phi: 0.6225 | Regression_energy: 1.8284 | Regression_eta: 0.6313 | Regression_pt: 1.9545 | Regression_sin_phi: 0.4892 | LR: 5.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6602s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3106 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4873.2 | uss_MB: 4102.0 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:24:51,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:31,569] INFO : Step 26700/50000 rank0 | Train Loss: 8.9932 | Train Loss Unweighted: 2.7662 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4208 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1507 | Regression_eta: 0.0088 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8187 | Classification_binary: 2.4208 | Regression_cos_phi: 0.5504 | Regression_energy: 2.0846 | Regression_eta: 0.4315 | Regression_pt: 2.2673 | Regression_sin_phi: 0.4200 | LR: 5.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6988s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3113 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4873.2 | uss_MB: 4102.1 | child_rss_MB: 3988.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:25:31,783] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:26:11,559] INFO : Step 26800/50000 rank0 | Train Loss: 9.0570 | Train Loss Unweighted: 2.8307 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5089 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1347 | Regression_eta: 0.0107 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9160 | Classification_binary: 2.5089 | Regression_cos_phi: 0.6399 | Regression_energy: 1.8634 | Regression_eta: 0.5249 | Regression_pt: 2.0282 | Regression_sin_phi: 0.5757 | LR: 4.99e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6625s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3091 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.3769 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4873.2 | uss_MB: 4102.1 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:26:11,774] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:26:51,686] INFO : Step 26900/50000 rank0 | Train Loss: 9.1927 | Train Loss Unweighted: 2.8242 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4926 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1421 | Regression_eta: 0.0125 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8390 | Classification_binary: 2.4926 | Regression_cos_phi: 0.7393 | Regression_energy: 1.9647 | Regression_eta: 0.6162 | Regression_pt: 2.0700 | Regression_sin_phi: 0.4707 | LR: 4.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1845 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4872.1 | uss_MB: 4100.7 | child_rss_MB: 3985.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:26:51,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:31,144] INFO : Step 27000/50000 rank0 | Train Loss: 9.5774 | Train Loss Unweighted: 2.9421 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5997 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1458 | Regression_eta: 0.0152 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8953 | Classification_binary: 2.5997 | Regression_cos_phi: 0.6992 | Regression_energy: 2.0164 | Regression_eta: 0.7490 | Regression_pt: 2.1167 | Regression_sin_phi: 0.5011 | LR: 4.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5541s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0607 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4873.1 | uss_MB: 4101.8 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:27:31,358] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:31,377] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 14:27:31,546] INFO : Running validation on rank0 +rank0 [2026-10-28 14:27:31,546] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:27:31,546] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:27:32,841] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 14:27:32,985] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:27:33,116] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:27:33,190] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:27:33,258] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:27:33,327] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:27:33,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:27:33,464] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:27:33,577] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:27:33,707] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:27:33,838] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:27:33,909] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:27:33,978] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:27:34,046] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:27:34,115] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:27:34,184] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:27:34,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:27:34,286] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:27:34,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:27:34,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:27:34,292] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:27:34,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:27:34,298] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:27:34,301] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:27:34,303] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:27:34,306] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:27:34,309] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:27:34,312] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:27:34,313] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:27:34,316] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:27:34,319] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:27:34,322] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:27:41,639] INFO : VALIDATION | Step=27000/50000 | Train Loss=9.5774 | Valid Loss=8.4622 | Train Loss Unweighted=2.9421 | Valid Loss Unweighted=2.7511 | Stale=0 | ETA=166.2m +rank0 [2026-10-28 14:27:41,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:41,709] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:27:41,712] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:41,998] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:43,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:44,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:45,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:45,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:45,937] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:27:45,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:46,015] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:27:46,018] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:46,102] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:47,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:48,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:49,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:49,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:49,985] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:27:49,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:50,074] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:27:50,076] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:27:50,152] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:51,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:52,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:53,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:54,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:54,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:27:54,146] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:54,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:54,848] INFO : Plotted number of elements +rank0 [2026-10-28 14:27:56,271] INFO : Plotted jets +rank0 [2026-10-28 14:27:57,894] INFO : Plotted jet ratio +rank0 [2026-10-28 14:27:57,927] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:58,156] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:58,529] INFO : Plotted number of elements +rank0 [2026-10-28 14:27:59,908] INFO : Plotted jets +rank0 [2026-10-28 14:28:00,970] INFO : Plotted jet ratio +rank0 [2026-10-28 14:28:01,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 14:28:01,207] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 14:28:01,630] INFO : Plotted number of elements +rank0 [2026-10-28 14:28:03,016] INFO : Plotted jets +rank0 [2026-10-28 14:28:04,123] INFO : Plotted jet ratio +rank0 [2026-10-28 14:28:43,777] INFO : Step 27100/50000 rank0 | Train Loss: 8.3222 | Train Loss Unweighted: 2.6726 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3747 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1265 | Regression_eta: 0.0093 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8360 | Classification_binary: 2.3747 | Regression_cos_phi: 0.5499 | Regression_energy: 1.7490 | Regression_eta: 0.4553 | Regression_pt: 1.8922 | Regression_sin_phi: 0.4652 | LR: 4.91e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7290s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1044 | max_loss: 0.0037 | max_backward: 0.3473 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3521 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5066.2 | uss_MB: 4301.1 | child_rss_MB: 3984.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:28:43,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:29:23,211] INFO : Step 27200/50000 rank0 | Train Loss: 8.2692 | Train Loss Unweighted: 2.6978 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4050 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1232 | Regression_eta: 0.0085 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8266 | Classification_binary: 2.4050 | Regression_cos_phi: 0.5257 | Regression_energy: 1.7040 | Regression_eta: 0.4176 | Regression_pt: 1.8712 | Regression_sin_phi: 0.5190 | LR: 4.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1331 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5066.2 | uss_MB: 4299.0 | child_rss_MB: 3982.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:29:23,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:03,187] INFO : Step 27300/50000 rank0 | Train Loss: 7.9742 | Train Loss Unweighted: 2.6823 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4038 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1192 | Regression_eta: 0.0099 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7716 | Classification_binary: 2.4038 | Regression_cos_phi: 0.4863 | Regression_energy: 1.6482 | Regression_eta: 0.4843 | Regression_pt: 1.7466 | Regression_sin_phi: 0.4334 | LR: 4.85e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3020 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4941.5 | uss_MB: 4174.1 | child_rss_MB: 3986.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:03,403] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:43,290] INFO : Step 27400/50000 rank0 | Train Loss: 8.3295 | Train Loss Unweighted: 2.7295 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4511 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1143 | Regression_eta: 0.0109 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8903 | Classification_binary: 2.4511 | Regression_cos_phi: 0.6428 | Regression_energy: 1.5804 | Regression_eta: 0.5361 | Regression_pt: 1.6998 | Regression_sin_phi: 0.5289 | LR: 4.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 0.2990 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4942.5 | uss_MB: 4175.2 | child_rss_MB: 3984.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:43,506] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:31:23,275] INFO : Step 27500/50000 rank0 | Train Loss: 8.0114 | Train Loss Unweighted: 2.6226 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.3414 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1191 | Regression_eta: 0.0079 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8256 | Classification_binary: 2.3414 | Regression_cos_phi: 0.4837 | Regression_energy: 1.6472 | Regression_eta: 0.3895 | Regression_pt: 1.7675 | Regression_sin_phi: 0.5566 | LR: 4.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1190 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4942.5 | uss_MB: 4175.4 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:31:23,489] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:03,015] INFO : Step 27600/50000 rank0 | Train Loss: 8.8212 | Train Loss Unweighted: 2.7518 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4465 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1278 | Regression_eta: 0.0134 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8434 | Classification_binary: 2.4465 | Regression_cos_phi: 0.7207 | Regression_energy: 1.7676 | Regression_eta: 0.6600 | Regression_pt: 1.8677 | Regression_sin_phi: 0.5153 | LR: 4.77e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5487s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0157 | max_backward: 0.2037 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3247 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4942.5 | uss_MB: 4175.6 | child_rss_MB: 3981.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 14:32:03,232] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:43,166] INFO : Step 27700/50000 rank0 | Train Loss: 7.7818 | Train Loss Unweighted: 2.6418 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3699 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1141 | Regression_eta: 0.0083 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8339 | Classification_binary: 2.3699 | Regression_cos_phi: 0.4699 | Regression_energy: 1.5780 | Regression_eta: 0.4066 | Regression_pt: 1.7442 | Regression_sin_phi: 0.3793 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0050 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3056 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4942.5 | uss_MB: 4175.7 | child_rss_MB: 4003.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:32:43,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:33:22,842] INFO : Step 27800/50000 rank0 | Train Loss: 8.3880 | Train Loss Unweighted: 2.7172 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4061 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1347 | Regression_eta: 0.0081 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7885 | Classification_binary: 2.4061 | Regression_cos_phi: 0.4675 | Regression_energy: 1.8621 | Regression_eta: 0.3974 | Regression_pt: 2.0287 | Regression_sin_phi: 0.4378 | LR: 4.71e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5808s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1288 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4942.5 | uss_MB: 4175.8 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:33:23,059] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:03,071] INFO : Step 27900/50000 rank0 | Train Loss: 8.0460 | Train Loss Unweighted: 2.6968 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4187 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1170 | Regression_eta: 0.0100 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7294 | Classification_binary: 2.4187 | Regression_cos_phi: 0.5654 | Regression_energy: 1.6181 | Regression_eta: 0.4890 | Regression_pt: 1.7626 | Regression_sin_phi: 0.4629 | LR: 4.68e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6406s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0353 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3103 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4941.1 | uss_MB: 4174.1 | child_rss_MB: 4007.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:03,283] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:42,693] INFO : Step 28000/50000 rank0 | Train Loss: 8.4309 | Train Loss Unweighted: 2.7268 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4223 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1295 | Regression_eta: 0.0094 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8464 | Classification_binary: 2.4223 | Regression_cos_phi: 0.4994 | Regression_energy: 1.7907 | Regression_eta: 0.4643 | Regression_pt: 1.9546 | Regression_sin_phi: 0.4532 | LR: 4.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6453s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2511 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3954 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4941.1 | uss_MB: 4175.3 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:42,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:42,970] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 14:34:43,143] INFO : Running validation on rank0 +rank0 [2026-10-28 14:34:43,143] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:34:43,143] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:34:44,444] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 14:34:44,590] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:34:44,719] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:34:44,792] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:34:44,859] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:34:44,927] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:34:44,997] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:34:45,065] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:34:45,181] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:34:45,312] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:34:45,443] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:34:45,514] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:34:45,582] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:34:45,649] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:34:45,717] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:34:45,787] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:34:45,885] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:34:45,892] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:34:45,894] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:34:45,895] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:34:45,898] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:34:45,900] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:34:45,902] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:34:45,904] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:34:45,905] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:34:45,908] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:34:45,909] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:34:45,912] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:34:45,913] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:34:45,915] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:34:45,916] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:34:45,919] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:34:53,360] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.4309 | Valid Loss=8.4196 | Train Loss Unweighted=2.7268 | Valid Loss Unweighted=2.7364 | Stale=0 | ETA=159.0m +rank0 [2026-10-28 14:34:53,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:53,432] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:34:53,436] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:34:53,603] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:34:55,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:34:56,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:34:56,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:34:57,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:34:57,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:34:57,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:57,837] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:34:57,840] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:34:57,945] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:34:59,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:35:00,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:35:01,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:35:01,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:35:01,889] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:35:01,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:35:01,946] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:35:01,949] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:35:02,054] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:35:03,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:35:04,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:35:05,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:35:05,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:35:05,975] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:35:05,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:35:06,237] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:35:06,686] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:08,112] INFO : Plotted jets +rank0 [2026-10-28 14:35:09,739] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:09,772] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:35:09,998] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:35:10,371] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:11,745] INFO : Plotted jets +rank0 [2026-10-28 14:35:12,812] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:12,847] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 14:35:13,052] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 14:35:13,463] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:14,844] INFO : Plotted jets +rank0 [2026-10-28 14:35:15,940] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:56,473] INFO : Step 28100/50000 rank0 | Train Loss: 7.9455 | Train Loss Unweighted: 2.7135 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4332 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1205 | Regression_eta: 0.0077 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7746 | Classification_binary: 2.4332 | Regression_cos_phi: 0.4995 | Regression_energy: 1.6666 | Regression_eta: 0.3792 | Regression_pt: 1.7914 | Regression_sin_phi: 0.4010 | LR: 4.63e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0034 | max_backward: 0.2035 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5133.5 | uss_MB: 4367.7 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:35:56,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:36,546] INFO : Step 28200/50000 rank0 | Train Loss: 8.6838 | Train Loss Unweighted: 2.8756 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5627 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1352 | Regression_eta: 0.0083 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8806 | Classification_binary: 2.5627 | Regression_cos_phi: 0.5001 | Regression_energy: 1.8698 | Regression_eta: 0.4061 | Regression_pt: 1.9964 | Regression_sin_phi: 0.4680 | LR: 4.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0845 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3108 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5133.5 | uss_MB: 4367.7 | child_rss_MB: 4007.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:36,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:17,137] INFO : Step 28300/50000 rank0 | Train Loss: 8.1447 | Train Loss Unweighted: 2.8411 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5674 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1168 | Regression_eta: 0.0093 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8325 | Classification_binary: 2.5674 | Regression_cos_phi: 0.5565 | Regression_energy: 1.6151 | Regression_eta: 0.4590 | Regression_pt: 1.6806 | Regression_sin_phi: 0.4335 | LR: 4.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6289s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0805 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.3056 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5014.0 | uss_MB: 4247.1 | child_rss_MB: 3984.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:37:17,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:57,283] INFO : Step 28400/50000 rank0 | Train Loss: 8.8517 | Train Loss Unweighted: 2.7778 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4528 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1410 | Regression_eta: 0.0107 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.4528 | Regression_cos_phi: 0.5220 | Regression_energy: 1.9497 | Regression_eta: 0.5246 | Regression_pt: 2.0476 | Regression_sin_phi: 0.4657 | LR: 4.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6771s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0628 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3107 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3042 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5016.0 | uss_MB: 4248.2 | child_rss_MB: 4013.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:37:57,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:37,084] INFO : Step 28500/50000 rank0 | Train Loss: 8.5647 | Train Loss Unweighted: 2.7590 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4571 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1277 | Regression_eta: 0.0084 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.4571 | Regression_cos_phi: 0.6435 | Regression_energy: 1.7656 | Regression_eta: 0.4126 | Regression_pt: 1.9063 | Regression_sin_phi: 0.5578 | LR: 4.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6478s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1389 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5016.0 | uss_MB: 4248.4 | child_rss_MB: 3984.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:38:37,312] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:17,519] INFO : Step 28600/50000 rank0 | Train Loss: 8.1791 | Train Loss Unweighted: 2.7552 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4752 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1186 | Regression_eta: 0.0094 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7609 | Classification_binary: 2.4752 | Regression_cos_phi: 0.5793 | Regression_energy: 1.6402 | Regression_eta: 0.4599 | Regression_pt: 1.7473 | Regression_sin_phi: 0.5164 | LR: 4.49e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0443 | max_backward: 0.2032 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3227 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:39:17,735] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:57,500] INFO : Step 28700/50000 rank0 | Train Loss: 8.8816 | Train Loss Unweighted: 2.7869 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4682 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1363 | Regression_eta: 0.0114 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8648 | Classification_binary: 2.4682 | Regression_cos_phi: 0.5636 | Regression_energy: 1.8846 | Regression_eta: 0.5625 | Regression_pt: 1.9893 | Regression_sin_phi: 0.5485 | LR: 4.46e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6124s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0255 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3114 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:39:57,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:37,751] INFO : Step 28800/50000 rank0 | Train Loss: 8.0992 | Train Loss Unweighted: 2.5726 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.2823 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1228 | Regression_eta: 0.0097 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7673 | Classification_binary: 2.2823 | Regression_cos_phi: 0.5703 | Regression_energy: 1.6982 | Regression_eta: 0.4764 | Regression_pt: 1.8494 | Regression_sin_phi: 0.4553 | LR: 4.44e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7141s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3114 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:40:37,964] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:17,229] INFO : Step 28900/50000 rank0 | Train Loss: 8.1676 | Train Loss Unweighted: 2.6658 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3724 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1264 | Regression_eta: 0.0099 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7497 | Classification_binary: 2.3724 | Regression_cos_phi: 0.4947 | Regression_energy: 1.7478 | Regression_eta: 0.4882 | Regression_pt: 1.8642 | Regression_sin_phi: 0.4506 | LR: 4.41e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.8 | uss_MB: 4247.0 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:41:17,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:57,128] INFO : Step 29000/50000 rank0 | Train Loss: 8.1378 | Train Loss Unweighted: 2.6649 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3877 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1162 | Regression_eta: 0.0107 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.3877 | Regression_cos_phi: 0.5456 | Regression_energy: 1.6063 | Regression_eta: 0.5244 | Regression_pt: 1.7019 | Regression_sin_phi: 0.5724 | LR: 4.38e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0724 | max_backward: 0.2028 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3270 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5016.8 | uss_MB: 4248.1 | child_rss_MB: 3987.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:41:57,340] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:57,382] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 14:41:57,558] INFO : Running validation on rank0 +rank0 [2026-10-28 14:41:57,558] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:41:57,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:41:58,883] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 14:41:59,028] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:41:59,157] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:41:59,230] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:41:59,298] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:41:59,365] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:41:59,434] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:41:59,502] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:41:59,615] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:41:59,745] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:41:59,875] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:41:59,946] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:42:00,013] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:42:00,081] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:42:00,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:42:00,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:42:00,316] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:42:00,322] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:42:00,325] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:42:00,328] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:42:00,329] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:42:00,331] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:42:00,334] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:42:00,335] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:42:00,337] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:42:00,340] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:42:00,341] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:42:00,343] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:42:00,344] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:42:00,346] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:42:00,347] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:42:00,350] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:42:07,770] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.1378 | Valid Loss=8.4468 | Train Loss Unweighted=2.6649 | Valid Loss Unweighted=2.7471 | Stale=1 | ETA=151.8m +rank0 [2026-10-28 14:42:07,781] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:07,843] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:42:07,847] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:08,014] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:09,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:10,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:11,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:12,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:12,194] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:12,198] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:12,253] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:42:12,256] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:12,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:14,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:14,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:15,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:16,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:16,352] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:16,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:16,409] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:42:16,412] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:42:16,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:18,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:18,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:19,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:20,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:20,476] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:20,483] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:20,741] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:21,190] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:22,610] INFO : Plotted jets +rank0 [2026-10-28 14:42:24,229] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:24,262] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:24,488] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:24,861] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:26,236] INFO : Plotted jets +rank0 [2026-10-28 14:42:27,295] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:27,327] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 14:42:27,531] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 14:42:27,942] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:29,321] INFO : Plotted jets +rank0 [2026-10-28 14:42:30,425] INFO : Plotted jet ratio +rank0 [2026-10-28 14:43:10,094] INFO : Step 29100/50000 rank0 | Train Loss: 8.0370 | Train Loss Unweighted: 2.6942 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4143 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1194 | Regression_eta: 0.0090 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.4143 | Regression_cos_phi: 0.5858 | Regression_energy: 1.6510 | Regression_eta: 0.4424 | Regression_pt: 1.7641 | Regression_sin_phi: 0.4294 | LR: 4.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0575 | max_backward: 0.2030 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3150 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5209.6 | uss_MB: 4441.9 | child_rss_MB: 3984.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 14:43:10,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:43:49,655] INFO : Step 29200/50000 rank0 | Train Loss: 8.6408 | Train Loss Unweighted: 2.7512 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4435 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1316 | Regression_eta: 0.0102 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8166 | Classification_binary: 2.4435 | Regression_cos_phi: 0.6314 | Regression_energy: 1.8195 | Regression_eta: 0.4991 | Regression_pt: 1.9296 | Regression_sin_phi: 0.5011 | LR: 4.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5963s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.2381 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5209.6 | uss_MB: 4441.9 | child_rss_MB: 3985.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:43:49,881] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:44:30,789] INFO : Step 29300/50000 rank0 | Train Loss: 8.7085 | Train Loss Unweighted: 2.8323 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5252 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1285 | Regression_eta: 0.0096 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8329 | Classification_binary: 2.5252 | Regression_cos_phi: 0.5330 | Regression_energy: 1.7776 | Regression_eta: 0.4728 | Regression_pt: 1.9635 | Regression_sin_phi: 0.6036 | LR: 4.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0516 | max_backward: 0.2030 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.3235 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4321.2 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:44:31,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:10,803] INFO : Step 29400/50000 rank0 | Train Loss: 8.5965 | Train Loss Unweighted: 2.7093 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4049 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1288 | Regression_eta: 0.0112 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 2.4049 | Regression_cos_phi: 0.6475 | Regression_energy: 1.7806 | Regression_eta: 0.5525 | Regression_pt: 1.9038 | Regression_sin_phi: 0.4811 | LR: 4.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5439s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0187 | max_backward: 0.2035 | max_optimizer_step: 0.1333 | max_tensorboard_logging: 0.3153 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4322.5 | child_rss_MB: 3987.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:11,016] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:50,347] INFO : Step 29500/50000 rank0 | Train Loss: 8.5762 | Train Loss Unweighted: 2.6817 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3742 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1318 | Regression_eta: 0.0100 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8382 | Classification_binary: 2.3742 | Regression_cos_phi: 0.6068 | Regression_energy: 1.8231 | Regression_eta: 0.4925 | Regression_pt: 1.9150 | Regression_sin_phi: 0.5264 | LR: 4.24e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1606 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 3984.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:45:50,563] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:46:29,657] INFO : Step 29600/50000 rank0 | Train Loss: 8.7316 | Train Loss Unweighted: 2.7767 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4712 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1294 | Regression_eta: 0.0097 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8534 | Classification_binary: 2.4712 | Regression_cos_phi: 0.6981 | Regression_energy: 1.7887 | Regression_eta: 0.4775 | Regression_pt: 1.8950 | Regression_sin_phi: 0.5477 | LR: 4.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0814 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:46:29,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:09,745] INFO : Step 29700/50000 rank0 | Train Loss: 8.3128 | Train Loss Unweighted: 2.6743 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3783 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1256 | Regression_eta: 0.0099 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8165 | Classification_binary: 2.3783 | Regression_cos_phi: 0.5460 | Regression_energy: 1.7366 | Regression_eta: 0.4875 | Regression_pt: 1.8720 | Regression_sin_phi: 0.4758 | LR: 4.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6223s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0132 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.2980 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 4015.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:09,958] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:50,576] INFO : Step 29800/50000 rank0 | Train Loss: 8.1765 | Train Loss Unweighted: 2.7690 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4840 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1225 | Regression_eta: 0.0098 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7914 | Classification_binary: 2.4840 | Regression_cos_phi: 0.5314 | Regression_energy: 1.6946 | Regression_eta: 0.4814 | Regression_pt: 1.7851 | Regression_sin_phi: 0.4087 | LR: 4.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0644 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3074 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5089.6 | uss_MB: 4322.8 | child_rss_MB: 3988.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:50,792] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:48:30,607] INFO : Step 29900/50000 rank0 | Train Loss: 8.0130 | Train Loss Unweighted: 2.7386 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4602 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1187 | Regression_eta: 0.0089 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.4602 | Regression_cos_phi: 0.5121 | Regression_energy: 1.6420 | Regression_eta: 0.4363 | Regression_pt: 1.7674 | Regression_sin_phi: 0.4617 | LR: 4.14e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0932 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3061 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5088.6 | uss_MB: 4321.1 | child_rss_MB: 4010.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:48:30,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:10,833] INFO : Step 30000/50000 rank0 | Train Loss: 7.9866 | Train Loss Unweighted: 2.6757 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3953 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1209 | Regression_eta: 0.0085 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8436 | Classification_binary: 2.3953 | Regression_cos_phi: 0.4803 | Regression_energy: 1.6718 | Regression_eta: 0.4195 | Regression_pt: 1.7476 | Regression_sin_phi: 0.4286 | LR: 4.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3066 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5089.6 | uss_MB: 4322.4 | child_rss_MB: 3985.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:49:11,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:11,064] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 14:49:11,357] INFO : Running validation on rank0 +rank0 [2026-10-28 14:49:11,357] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:49:11,357] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:49:12,664] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 14:49:12,807] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:49:12,937] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:49:13,010] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:49:13,078] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:49:13,145] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:49:13,213] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:49:13,282] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:49:13,395] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:49:13,525] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:49:13,655] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:49:13,726] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:49:13,795] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:49:13,863] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:49:13,932] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:49:14,002] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:49:14,100] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:49:14,107] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:49:14,109] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:49:14,110] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:49:14,113] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:49:14,114] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:49:14,117] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:49:14,119] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:49:14,120] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:49:14,123] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:49:14,124] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:49:14,126] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:49:14,129] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:49:14,129] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:49:14,132] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:49:14,134] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:49:21,502] INFO : VALIDATION | Step=30000/50000 | Train Loss=7.9866 | Valid Loss=8.3367 | Train Loss Unweighted=2.6757 | Valid Loss Unweighted=2.7139 | Stale=0 | ETA=144.5m +rank0 [2026-10-28 14:49:21,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:21,572] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:49:21,575] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:21,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:23,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:24,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:24,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:25,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:25,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:25,816] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:25,872] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:49:25,875] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:25,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:27,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:28,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:29,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:29,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:30,021] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:30,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:30,079] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:49:30,082] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:49:30,187] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:32,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:32,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:33,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:34,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:34,179] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:34,187] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:34,444] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:34,892] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:36,320] INFO : Plotted jets +rank0 [2026-10-28 14:49:37,424] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:37,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:37,677] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:38,045] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:39,968] INFO : Plotted jets +rank0 [2026-10-28 14:49:41,041] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:41,073] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 14:49:41,277] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 14:49:41,689] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:43,073] INFO : Plotted jets +rank0 [2026-10-28 14:49:44,175] INFO : Plotted jet ratio +rank0 [2026-10-28 14:50:23,997] INFO : Step 30100/50000 rank0 | Train Loss: 8.0665 | Train Loss Unweighted: 2.6752 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3797 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1286 | Regression_eta: 0.0079 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7370 | Classification_binary: 2.3797 | Regression_cos_phi: 0.4569 | Regression_energy: 1.7789 | Regression_eta: 0.3883 | Regression_pt: 1.9149 | Regression_sin_phi: 0.4109 | LR: 4.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1354 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5285.6 | uss_MB: 4521.1 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:50:24,216] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:04,584] INFO : Step 30200/50000 rank0 | Train Loss: 8.1842 | Train Loss Unweighted: 2.7650 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4895 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1134 | Regression_eta: 0.0105 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8585 | Classification_binary: 2.4895 | Regression_cos_phi: 0.5345 | Regression_energy: 1.5686 | Regression_eta: 0.5154 | Regression_pt: 1.7216 | Regression_sin_phi: 0.4961 | LR: 4.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6242s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0783 | max_backward: 0.2036 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3229 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5285.6 | uss_MB: 4521.2 | child_rss_MB: 3987.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:04,797] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:44,963] INFO : Step 30300/50000 rank0 | Train Loss: 8.9950 | Train Loss Unweighted: 2.8466 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5274 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1365 | Regression_eta: 0.0101 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8733 | Classification_binary: 2.5274 | Regression_cos_phi: 0.6648 | Regression_energy: 1.8876 | Regression_eta: 0.4943 | Regression_pt: 1.9843 | Regression_sin_phi: 0.5633 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0435 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3047 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.2936 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5285.6 | uss_MB: 4521.2 | child_rss_MB: 4012.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:45,178] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:52:25,255] INFO : Step 30400/50000 rank0 | Train Loss: 8.3280 | Train Loss Unweighted: 2.7375 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4297 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1344 | Regression_eta: 0.0073 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8246 | Classification_binary: 2.4297 | Regression_cos_phi: 0.4167 | Regression_energy: 1.8589 | Regression_eta: 0.3607 | Regression_pt: 1.9914 | Regression_sin_phi: 0.4461 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6555s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2239 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5166.6 | uss_MB: 4399.5 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:52:25,469] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:05,217] INFO : Step 30500/50000 rank0 | Train Loss: 8.2259 | Train Loss Unweighted: 2.7119 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4219 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1242 | Regression_eta: 0.0088 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8187 | Classification_binary: 2.4219 | Regression_cos_phi: 0.5728 | Regression_energy: 1.7178 | Regression_eta: 0.4328 | Regression_pt: 1.8229 | Regression_sin_phi: 0.4390 | LR: 3.98e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6970s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3097 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5167.6 | uss_MB: 4399.7 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:53:05,429] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:45,509] INFO : Step 30600/50000 rank0 | Train Loss: 8.4804 | Train Loss Unweighted: 2.8380 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5315 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1322 | Regression_eta: 0.0090 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8260 | Classification_binary: 2.5315 | Regression_cos_phi: 0.3949 | Regression_energy: 1.8284 | Regression_eta: 0.4398 | Regression_pt: 1.9722 | Regression_sin_phi: 0.4875 | LR: 3.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7204s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1797 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5167.6 | uss_MB: 4399.9 | child_rss_MB: 3981.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:53:45,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:54:25,640] INFO : Step 30700/50000 rank0 | Train Loss: 8.3313 | Train Loss Unweighted: 2.7364 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4408 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1266 | Regression_eta: 0.0112 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7721 | Classification_binary: 2.4408 | Regression_cos_phi: 0.5403 | Regression_energy: 1.7504 | Regression_eta: 0.5521 | Regression_pt: 1.8679 | Regression_sin_phi: 0.4078 | LR: 3.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6736s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0384 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3102 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5167.6 | uss_MB: 4400.0 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:54:25,855] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:06,039] INFO : Step 30800/50000 rank0 | Train Loss: 8.3344 | Train Loss Unweighted: 2.6122 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3147 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1261 | Regression_eta: 0.0099 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7661 | Classification_binary: 2.3147 | Regression_cos_phi: 0.5546 | Regression_energy: 1.7431 | Regression_eta: 0.4853 | Regression_pt: 1.8736 | Regression_sin_phi: 0.5969 | LR: 3.90e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0885 | max_backward: 0.2031 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3259 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5167.6 | uss_MB: 4400.1 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:55:06,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:46,639] INFO : Step 30900/50000 rank0 | Train Loss: 8.2921 | Train Loss Unweighted: 2.7752 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4775 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1283 | Regression_eta: 0.0078 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7970 | Classification_binary: 2.4775 | Regression_cos_phi: 0.4953 | Regression_energy: 1.7743 | Regression_eta: 0.3812 | Regression_pt: 1.9164 | Regression_sin_phi: 0.4504 | LR: 3.87e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3094 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5167.6 | uss_MB: 4400.1 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:55:46,851] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:56:26,824] INFO : Step 31000/50000 rank0 | Train Loss: 7.9653 | Train Loss Unweighted: 2.6458 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3765 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1117 | Regression_eta: 0.0084 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7758 | Classification_binary: 2.3765 | Regression_cos_phi: 0.6365 | Regression_energy: 1.5451 | Regression_eta: 0.4142 | Regression_pt: 1.6802 | Regression_sin_phi: 0.5370 | LR: 3.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6019s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1783 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5168.4 | uss_MB: 4399.7 | child_rss_MB: 3984.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:56:27,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:56:27,060] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 14:56:27,232] INFO : Running validation on rank0 +rank0 [2026-10-28 14:56:27,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:56:27,233] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:56:28,531] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 14:56:28,676] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:56:28,805] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:56:28,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:56:28,946] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:56:29,013] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:56:29,082] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:56:29,151] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:56:29,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:56:29,394] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:56:29,525] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:56:29,595] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:56:29,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:56:29,732] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:56:29,800] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:56:29,869] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:56:29,966] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:56:29,971] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:56:29,974] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:56:29,975] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:56:29,978] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:56:29,980] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:56:29,982] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:56:29,984] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:56:29,985] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:56:29,988] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:56:29,989] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:56:29,991] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:56:29,993] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:56:29,995] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:56:29,996] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:56:29,998] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:56:37,286] INFO : VALIDATION | Step=31000/50000 | Train Loss=7.9653 | Valid Loss=8.3178 | Train Loss Unweighted=2.6458 | Valid Loss Unweighted=2.7033 | Stale=0 | ETA=137.3m +rank0 [2026-10-28 14:56:37,297] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:37,360] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:56:37,363] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:37,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:39,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:40,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:40,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:41,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:41,776] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:56:41,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:41,835] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:56:41,838] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:41,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:43,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:44,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:45,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:45,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:45,861] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:56:45,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:45,919] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:56:45,922] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:56:46,030] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:48,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:48,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:49,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:49,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:50,057] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:56:50,065] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:50,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:50,786] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:52,212] INFO : Plotted jets +rank0 [2026-10-28 14:56:53,317] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:53,353] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:53,574] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:53,948] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:55,900] INFO : Plotted jets +rank0 [2026-10-28 14:56:56,964] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:56,997] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 14:56:57,201] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 14:56:57,613] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:58,993] INFO : Plotted jets +rank0 [2026-10-28 14:57:00,099] INFO : Plotted jet ratio +rank0 [2026-10-28 14:57:40,686] INFO : Step 31100/50000 rank0 | Train Loss: 8.0023 | Train Loss Unweighted: 2.5771 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.3065 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1116 | Regression_eta: 0.0109 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.3065 | Regression_cos_phi: 0.6045 | Regression_energy: 1.5427 | Regression_eta: 0.5339 | Regression_pt: 1.6521 | Regression_sin_phi: 0.5037 | LR: 3.82e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2128 | max_loss: 0.0035 | max_backward: 0.3951 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5348.0 | uss_MB: 4583.2 | child_rss_MB: 4008.5 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 14:57:40,900] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:58:20,370] INFO : Step 31200/50000 rank0 | Train Loss: 7.6452 | Train Loss Unweighted: 2.6647 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4094 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1061 | Regression_eta: 0.0089 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7630 | Classification_binary: 2.4094 | Regression_cos_phi: 0.4981 | Regression_energy: 1.4666 | Regression_eta: 0.4380 | Regression_pt: 1.5998 | Regression_sin_phi: 0.4702 | LR: 3.79e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2099 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5348.0 | uss_MB: 4581.1 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 14:58:20,582] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:00,444] INFO : Step 31300/50000 rank0 | Train Loss: 8.6953 | Train Loss Unweighted: 2.7277 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.4206 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1280 | Regression_eta: 0.0114 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9230 | Classification_binary: 2.4206 | Regression_cos_phi: 0.6289 | Regression_energy: 1.7694 | Regression_eta: 0.5591 | Regression_pt: 1.9274 | Regression_sin_phi: 0.4669 | LR: 3.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6430s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.2109 | max_optimizer_step: 0.1357 | max_tensorboard_logging: 0.3879 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5348.0 | uss_MB: 4581.2 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:59:00,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:40,022] INFO : Step 31400/50000 rank0 | Train Loss: 8.1009 | Train Loss Unweighted: 2.6607 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3561 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1349 | Regression_eta: 0.0078 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7300 | Classification_binary: 2.3561 | Regression_cos_phi: 0.3871 | Regression_energy: 1.8655 | Regression_eta: 0.3847 | Regression_pt: 1.9816 | Regression_sin_phi: 0.3959 | LR: 3.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1583 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5240.5 | uss_MB: 4473.1 | child_rss_MB: 3987.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:59:40,237] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:00:20,132] INFO : Step 31500/50000 rank0 | Train Loss: 8.3926 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4649 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1210 | Regression_eta: 0.0092 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7797 | Classification_binary: 2.4649 | Regression_cos_phi: 0.6431 | Regression_energy: 1.6739 | Regression_eta: 0.4525 | Regression_pt: 1.8460 | Regression_sin_phi: 0.5325 | LR: 3.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7576s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1074 | max_loss: 0.0037 | max_backward: 0.3929 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5240.5 | uss_MB: 4475.4 | child_rss_MB: 4015.0 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 15:00:20,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:00,401] INFO : Step 31600/50000 rank0 | Train Loss: 8.8795 | Train Loss Unweighted: 2.7464 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4256 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1359 | Regression_eta: 0.0111 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8785 | Classification_binary: 2.4256 | Regression_cos_phi: 0.6144 | Regression_energy: 1.8786 | Regression_eta: 0.5434 | Regression_pt: 2.0267 | Regression_sin_phi: 0.5121 | LR: 3.69e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0854 | max_backward: 0.2034 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3316 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5240.5 | uss_MB: 4473.4 | child_rss_MB: 3986.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:01:00,613] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:40,504] INFO : Step 31700/50000 rank0 | Train Loss: 8.0771 | Train Loss Unweighted: 2.5782 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2800 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1284 | Regression_eta: 0.0091 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7460 | Classification_binary: 2.2800 | Regression_cos_phi: 0.4495 | Regression_energy: 1.7757 | Regression_eta: 0.4487 | Regression_pt: 1.9300 | Regression_sin_phi: 0.4472 | LR: 3.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6171s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1810 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:01:40,724] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:20,793] INFO : Step 31800/50000 rank0 | Train Loss: 8.4507 | Train Loss Unweighted: 2.7522 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4542 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1276 | Regression_eta: 0.0104 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7872 | Classification_binary: 2.4542 | Regression_cos_phi: 0.5117 | Regression_energy: 1.7650 | Regression_eta: 0.5122 | Regression_pt: 1.8625 | Regression_sin_phi: 0.5579 | LR: 3.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6353s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0282 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:02:21,007] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:01,080] INFO : Step 31900/50000 rank0 | Train Loss: 7.9445 | Train Loss Unweighted: 2.6661 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3802 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1239 | Regression_eta: 0.0081 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7345 | Classification_binary: 2.3802 | Regression_cos_phi: 0.4251 | Regression_energy: 1.7133 | Regression_eta: 0.3965 | Regression_pt: 1.8340 | Regression_sin_phi: 0.4610 | LR: 3.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6382s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2430 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:03:01,294] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:41,217] INFO : Step 32000/50000 rank0 | Train Loss: 8.5018 | Train Loss Unweighted: 2.6727 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3696 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1288 | Regression_eta: 0.0089 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8339 | Classification_binary: 2.3696 | Regression_cos_phi: 0.7049 | Regression_energy: 1.7806 | Regression_eta: 0.4366 | Regression_pt: 1.9054 | Regression_sin_phi: 0.4707 | LR: 3.58e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3089 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5240.7 | uss_MB: 4473.2 | child_rss_MB: 3987.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 15:03:41,432] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:41,545] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 15:03:41,724] INFO : Running validation on rank0 +rank0 [2026-10-28 15:03:41,724] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:03:41,724] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:03:43,016] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 15:03:43,159] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:03:43,289] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:03:43,362] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:03:43,430] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:03:43,498] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:03:43,567] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:03:43,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:03:43,750] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:03:43,881] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:03:44,021] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:03:44,091] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:03:44,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:03:44,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:03:44,298] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:03:44,368] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:03:44,470] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:03:44,476] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:03:44,479] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:03:44,481] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:03:44,483] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:03:44,485] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:03:44,488] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:03:44,489] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:03:44,491] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:03:44,494] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:03:44,495] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:03:44,497] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:03:44,500] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:03:44,501] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:03:44,503] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:03:44,505] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:03:51,801] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.5018 | Valid Loss=8.3395 | Train Loss Unweighted=2.6727 | Valid Loss Unweighted=2.7105 | Stale=1 | ETA=130.1m +rank0 [2026-10-28 15:03:51,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:03:51,873] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:03:51,877] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:03:51,995] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:53,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:54,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:55,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:03:56,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:03:56,171] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:03:56,174] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:03:56,231] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:03:56,234] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:56,342] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:58,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:58,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:59,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:04:00,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:04:00,459] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:04:00,463] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:04:00,517] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:04:00,520] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:04:00,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:04:02,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:04:03,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:04:03,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:04:04,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:04:04,567] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:04:04,586] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:04:04,839] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:04:05,285] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:06,709] INFO : Plotted jets +rank0 [2026-10-28 15:04:07,808] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:07,840] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:04:08,651] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:04:09,024] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:10,401] INFO : Plotted jets +rank0 [2026-10-28 15:04:11,463] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:11,495] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 15:04:11,704] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 15:04:12,115] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:13,502] INFO : Plotted jets +rank0 [2026-10-28 15:04:14,602] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:55,036] INFO : Step 32100/50000 rank0 | Train Loss: 7.9407 | Train Loss Unweighted: 2.5889 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3074 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1200 | Regression_eta: 0.0083 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7660 | Classification_binary: 2.3074 | Regression_cos_phi: 0.5552 | Regression_energy: 1.6595 | Regression_eta: 0.4071 | Regression_pt: 1.7835 | Regression_sin_phi: 0.4620 | LR: 3.56e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.1025 | max_backward: 0.2033 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3093 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:04:55,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:05:34,905] INFO : Step 32200/50000 rank0 | Train Loss: 7.9920 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4130 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1228 | Regression_eta: 0.0078 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7902 | Classification_binary: 2.4130 | Regression_cos_phi: 0.4839 | Regression_energy: 1.6984 | Regression_eta: 0.3816 | Regression_pt: 1.8488 | Regression_sin_phi: 0.3761 | LR: 3.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7124s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0959 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.2769 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 4013.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:05:35,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:15,601] INFO : Step 32300/50000 rank0 | Train Loss: 8.1707 | Train Loss Unweighted: 2.7288 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4487 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1182 | Regression_eta: 0.0091 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8037 | Classification_binary: 2.4487 | Regression_cos_phi: 0.5587 | Regression_energy: 1.6345 | Regression_eta: 0.4493 | Regression_pt: 1.7423 | Regression_sin_phi: 0.5335 | LR: 3.51e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7584s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1578 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.3009 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:06:15,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:55,027] INFO : Step 32400/50000 rank0 | Train Loss: 7.8144 | Train Loss Unweighted: 2.6143 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3355 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1183 | Regression_eta: 0.0087 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7414 | Classification_binary: 2.3355 | Regression_cos_phi: 0.4452 | Regression_energy: 1.6355 | Regression_eta: 0.4262 | Regression_pt: 1.8069 | Regression_sin_phi: 0.4238 | LR: 3.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6507s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0324 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3114 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5312.8 | uss_MB: 4545.6 | child_rss_MB: 4013.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:06:55,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:07:34,657] INFO : Step 32500/50000 rank0 | Train Loss: 8.0604 | Train Loss Unweighted: 2.6533 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.3623 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1248 | Regression_eta: 0.0088 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 2.3623 | Regression_cos_phi: 0.4237 | Regression_energy: 1.7255 | Regression_eta: 0.4339 | Regression_pt: 1.8638 | Regression_sin_phi: 0.4252 | LR: 3.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6821s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2753 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.6 | child_rss_MB: 3987.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:07:34,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:14,387] INFO : Step 32600/50000 rank0 | Train Loss: 8.3240 | Train Loss Unweighted: 2.6827 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3834 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1272 | Regression_eta: 0.0087 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7891 | Classification_binary: 2.3834 | Regression_cos_phi: 0.5349 | Regression_energy: 1.7596 | Regression_eta: 0.4298 | Regression_pt: 1.9190 | Regression_sin_phi: 0.5082 | LR: 3.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0288 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3100 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.8 | child_rss_MB: 4013.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:08:14,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:54,658] INFO : Step 32700/50000 rank0 | Train Loss: 8.1799 | Train Loss Unweighted: 2.6389 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3514 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1216 | Regression_eta: 0.0121 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7713 | Classification_binary: 2.3514 | Regression_cos_phi: 0.5153 | Regression_energy: 1.6815 | Regression_eta: 0.5950 | Regression_pt: 1.7974 | Regression_sin_phi: 0.4680 | LR: 3.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3091 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3988.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:08:54,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:09:34,973] INFO : Step 32800/50000 rank0 | Train Loss: 8.7853 | Train Loss Unweighted: 2.7267 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3961 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1436 | Regression_eta: 0.0087 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7688 | Classification_binary: 2.3961 | Regression_cos_phi: 0.5319 | Regression_energy: 1.9857 | Regression_eta: 0.4284 | Regression_pt: 2.1485 | Regression_sin_phi: 0.5259 | LR: 3.38e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2152 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:09:35,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:14,396] INFO : Step 32900/50000 rank0 | Train Loss: 8.1033 | Train Loss Unweighted: 2.6600 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3720 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1240 | Regression_eta: 0.0097 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7356 | Classification_binary: 2.3720 | Regression_cos_phi: 0.5337 | Regression_energy: 1.7153 | Regression_eta: 0.4769 | Regression_pt: 1.8164 | Regression_sin_phi: 0.4534 | LR: 3.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6170s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0807 | max_backward: 0.2028 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3171 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3988.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:10:14,608] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:54,436] INFO : Step 33000/50000 rank0 | Train Loss: 8.3068 | Train Loss Unweighted: 2.6918 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3953 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1255 | Regression_eta: 0.0090 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7464 | Classification_binary: 2.3953 | Regression_cos_phi: 0.6605 | Regression_energy: 1.7350 | Regression_eta: 0.4426 | Regression_pt: 1.9100 | Regression_sin_phi: 0.4169 | LR: 3.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6209s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0904 | max_backward: 0.2031 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3130 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5313.8 | uss_MB: 4545.6 | child_rss_MB: 3984.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:10:54,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:54,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 15:10:54,861] INFO : Running validation on rank0 +rank0 [2026-10-28 15:10:54,862] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:10:54,862] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:10:56,160] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 15:10:56,304] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:10:56,435] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:10:56,508] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:10:56,576] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:10:56,644] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:10:56,713] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:10:56,782] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:10:56,897] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:10:57,028] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:10:57,159] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:10:57,229] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:10:57,298] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:10:57,366] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:10:57,435] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:10:57,505] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:10:57,604] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:10:57,611] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:10:57,614] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:10:57,617] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:10:57,618] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:10:57,619] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:10:57,622] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:10:57,625] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:10:57,627] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:10:57,628] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:10:57,631] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:10:57,632] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:10:57,634] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:10:57,637] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:10:57,639] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:10:57,640] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:11:04,955] INFO : VALIDATION | Step=33000/50000 | Train Loss=8.3068 | Valid Loss=8.1916 | Train Loss Unweighted=2.6918 | Valid Loss Unweighted=2.6784 | Stale=0 | ETA=122.9m +rank0 [2026-10-28 15:11:04,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:05,054] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:11:05,074] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:05,265] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:07,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:07,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:08,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:09,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:09,498] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:09,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:09,557] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:11:09,560] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:09,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:11,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:12,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:12,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:13,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:13,645] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:13,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:13,702] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:11:13,705] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:11:13,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:15,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:16,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:16,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:17,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:17,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:17,785] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:18,041] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:18,486] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:19,908] INFO : Plotted jets +rank0 [2026-10-28 15:11:21,549] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:21,583] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:21,806] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:22,183] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:23,553] INFO : Plotted jets +rank0 [2026-10-28 15:11:24,623] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:24,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 15:11:24,861] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 15:11:25,273] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:26,659] INFO : Plotted jets +rank0 [2026-10-28 15:11:27,756] INFO : Plotted jet ratio +rank0 [2026-10-28 15:12:07,972] INFO : Step 33100/50000 rank0 | Train Loss: 7.6590 | Train Loss Unweighted: 2.6479 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3785 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1163 | Regression_eta: 0.0070 | Regression_pt: 0.1141 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7784 | Classification_binary: 2.3785 | Regression_cos_phi: 0.4573 | Regression_energy: 1.6077 | Regression_eta: 0.3439 | Regression_pt: 1.7160 | Regression_sin_phi: 0.3772 | LR: 3.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6978s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2810 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3985.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:12:08,184] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:48,017] INFO : Step 33200/50000 rank0 | Train Loss: 8.1805 | Train Loss Unweighted: 2.6437 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3472 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1271 | Regression_eta: 0.0095 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7584 | Classification_binary: 2.3472 | Regression_cos_phi: 0.4494 | Regression_energy: 1.7571 | Regression_eta: 0.4662 | Regression_pt: 1.9007 | Regression_sin_phi: 0.5016 | LR: 3.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6619s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2331 | max_optimizer_step: 0.1372 | max_tensorboard_logging: 0.3859 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3983.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:12:48,232] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:13:28,132] INFO : Step 33300/50000 rank0 | Train Loss: 8.5542 | Train Loss Unweighted: 2.7013 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3852 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1352 | Regression_eta: 0.0090 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8126 | Classification_binary: 2.3852 | Regression_cos_phi: 0.5542 | Regression_energy: 1.8703 | Regression_eta: 0.4407 | Regression_pt: 2.0543 | Regression_sin_phi: 0.4368 | LR: 3.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5626s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2037 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2935 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3981.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:13:28,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:08,415] INFO : Step 33400/50000 rank0 | Train Loss: 7.3038 | Train Loss Unweighted: 2.5144 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2665 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1052 | Regression_eta: 0.0077 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.2665 | Regression_cos_phi: 0.4786 | Regression_energy: 1.4547 | Regression_eta: 0.3774 | Regression_pt: 1.5433 | Regression_sin_phi: 0.4572 | LR: 3.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6281s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0957 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 0.2902 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3983.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:14:08,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:49,056] INFO : Step 33500/50000 rank0 | Train Loss: 8.9086 | Train Loss Unweighted: 2.7366 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4129 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1371 | Regression_eta: 0.0105 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8640 | Classification_binary: 2.4129 | Regression_cos_phi: 0.6572 | Regression_energy: 1.8964 | Regression_eta: 0.5154 | Regression_pt: 2.0567 | Regression_sin_phi: 0.5058 | LR: 3.21e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3006 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3986.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:14:49,279] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:15:28,806] INFO : Step 33600/50000 rank0 | Train Loss: 8.1289 | Train Loss Unweighted: 2.7008 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4211 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1166 | Regression_eta: 0.0092 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8177 | Classification_binary: 2.4211 | Regression_cos_phi: 0.5345 | Regression_energy: 1.6119 | Regression_eta: 0.4512 | Regression_pt: 1.7635 | Regression_sin_phi: 0.5291 | LR: 3.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6001s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0661 | max_backward: 0.2033 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3981.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:15:29,020] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:08,336] INFO : Step 33700/50000 rank0 | Train Loss: 8.2775 | Train Loss Unweighted: 2.6506 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3472 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1300 | Regression_eta: 0.0098 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7738 | Classification_binary: 2.3472 | Regression_cos_phi: 0.4315 | Regression_energy: 1.7982 | Regression_eta: 0.4829 | Regression_pt: 1.9553 | Regression_sin_phi: 0.4885 | LR: 3.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7089s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1787 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3986.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:16:08,563] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:49,167] INFO : Step 33800/50000 rank0 | Train Loss: 8.1756 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4052 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1237 | Regression_eta: 0.0085 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8041 | Classification_binary: 2.4052 | Regression_cos_phi: 0.5856 | Regression_energy: 1.7112 | Regression_eta: 0.4191 | Regression_pt: 1.8167 | Regression_sin_phi: 0.4336 | LR: 3.14e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6019s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3984.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:16:49,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:29,959] INFO : Step 33900/50000 rank0 | Train Loss: 8.7197 | Train Loss Unweighted: 2.7571 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4359 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1380 | Regression_eta: 0.0084 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.4359 | Regression_cos_phi: 0.4954 | Regression_energy: 1.9078 | Regression_eta: 0.4130 | Regression_pt: 2.0701 | Regression_sin_phi: 0.5603 | LR: 3.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7372s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1812 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:17:30,176] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:18:09,775] INFO : Step 34000/50000 rank0 | Train Loss: 8.3398 | Train Loss Unweighted: 2.6338 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3324 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1288 | Regression_eta: 0.0105 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7968 | Classification_binary: 2.3324 | Regression_cos_phi: 0.5599 | Regression_energy: 1.7812 | Regression_eta: 0.5158 | Regression_pt: 1.9101 | Regression_sin_phi: 0.4437 | LR: 3.09e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0618 | max_backward: 0.2030 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3162 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5385.4 | uss_MB: 4616.6 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:18:09,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:18:10,022] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 15:18:10,197] INFO : Running validation on rank0 +rank0 [2026-10-28 15:18:10,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:18:10,197] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:18:11,479] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 15:18:11,623] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:18:11,753] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:18:11,827] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:18:11,896] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:18:11,964] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:18:12,033] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:18:12,101] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:18:12,215] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:18:12,346] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:18:12,477] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:18:12,547] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:18:12,616] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:18:12,684] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:18:12,753] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:18:12,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:18:12,922] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:18:12,929] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:18:12,931] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:18:12,932] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:18:12,935] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:18:12,938] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:18:12,939] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:18:12,941] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:18:12,942] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:18:12,945] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:18:12,948] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:18:12,951] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:18:12,952] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:18:12,954] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:18:12,957] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:18:12,958] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:18:20,271] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.3398 | Valid Loss=8.2081 | Train Loss Unweighted=2.6338 | Valid Loss Unweighted=2.6720 | Stale=1 | ETA=115.7m +rank0 [2026-10-28 15:18:20,281] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:20,342] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:18:20,345] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:20,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:22,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:23,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:23,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:24,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:24,877] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:18:24,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:24,936] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:18:24,940] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:25,044] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:26,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:27,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:28,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:28,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:28,935] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:18:28,939] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:28,993] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:18:28,996] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:18:29,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:31,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:31,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:32,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:32,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:33,121] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:18:33,129] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:33,386] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:33,835] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:35,259] INFO : Plotted jets +rank0 [2026-10-28 15:18:36,903] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:36,937] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:37,162] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:37,537] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:38,918] INFO : Plotted jets +rank0 [2026-10-28 15:18:39,981] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:40,013] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 15:18:40,217] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 15:18:40,632] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:42,020] INFO : Plotted jets +rank0 [2026-10-28 15:18:43,125] INFO : Plotted jet ratio +rank0 [2026-10-28 15:19:23,276] INFO : Step 34100/50000 rank0 | Train Loss: 7.8147 | Train Loss Unweighted: 2.6169 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3510 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1104 | Regression_eta: 0.0091 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8478 | Classification_binary: 2.3510 | Regression_cos_phi: 0.4979 | Regression_energy: 1.5270 | Regression_eta: 0.4453 | Regression_pt: 1.6576 | Regression_sin_phi: 0.4881 | LR: 3.06e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.7427s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3035 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:19:23,493] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:03,323] INFO : Step 34200/50000 rank0 | Train Loss: 8.2253 | Train Loss Unweighted: 2.5970 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3086 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1214 | Regression_eta: 0.0084 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.3086 | Regression_cos_phi: 0.6964 | Regression_energy: 1.6793 | Regression_eta: 0.4111 | Regression_pt: 1.7952 | Regression_sin_phi: 0.4835 | LR: 3.04e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6548s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.1020 | max_backward: 0.2031 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:20:03,539] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:43,213] INFO : Step 34300/50000 rank0 | Train Loss: 8.6300 | Train Loss Unweighted: 2.7350 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1252 | Regression_eta: 0.0116 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7899 | Classification_binary: 2.4355 | Regression_cos_phi: 0.6272 | Regression_energy: 1.7314 | Regression_eta: 0.5700 | Regression_pt: 1.8587 | Regression_sin_phi: 0.6172 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0659 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3101 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.2948 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3987.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:20:43,427] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:21:23,381] INFO : Step 34400/50000 rank0 | Train Loss: 8.7127 | Train Loss Unweighted: 2.6732 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.3643 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1292 | Regression_eta: 0.0132 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9060 | Classification_binary: 2.3643 | Regression_cos_phi: 0.6291 | Regression_energy: 1.7871 | Regression_eta: 0.6467 | Regression_pt: 1.9154 | Regression_sin_phi: 0.4640 | LR: 2.99e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6575s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2251 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:21:23,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:03,391] INFO : Step 34500/50000 rank0 | Train Loss: 7.5934 | Train Loss Unweighted: 2.5883 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3261 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1112 | Regression_eta: 0.0082 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7768 | Classification_binary: 2.3261 | Regression_cos_phi: 0.4518 | Regression_energy: 1.5384 | Regression_eta: 0.4038 | Regression_pt: 1.6472 | Regression_sin_phi: 0.4495 | LR: 2.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7627s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.4038 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:22:03,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:43,089] INFO : Step 34600/50000 rank0 | Train Loss: 8.8270 | Train Loss Unweighted: 2.6202 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2798 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1486 | Regression_eta: 0.0110 | Regression_pt: 0.1470 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7269 | Classification_binary: 2.2798 | Regression_cos_phi: 0.4910 | Regression_energy: 2.0546 | Regression_eta: 0.5385 | Regression_pt: 2.2112 | Regression_sin_phi: 0.5249 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7416s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2265 | max_tensorboard_logging: 0.2936 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5460.2 | uss_MB: 4692.2 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:43,307] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:23:23,554] INFO : Step 34700/50000 rank0 | Train Loss: 8.7026 | Train Loss Unweighted: 2.6519 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3344 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1366 | Regression_eta: 0.0112 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.3344 | Regression_cos_phi: 0.6261 | Regression_energy: 1.8885 | Regression_eta: 0.5524 | Regression_pt: 1.9903 | Regression_sin_phi: 0.5147 | LR: 2.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7350s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3981.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:23:23,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:04,369] INFO : Step 34800/50000 rank0 | Train Loss: 8.5479 | Train Loss Unweighted: 2.6638 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3466 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1376 | Regression_eta: 0.0083 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7799 | Classification_binary: 2.3466 | Regression_cos_phi: 0.5751 | Regression_energy: 1.9027 | Regression_eta: 0.4081 | Regression_pt: 2.0374 | Regression_sin_phi: 0.4982 | LR: 2.90e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1005 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3112 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:24:04,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:44,103] INFO : Step 34900/50000 rank0 | Train Loss: 8.1676 | Train Loss Unweighted: 2.6792 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3878 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1260 | Regression_eta: 0.0087 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7842 | Classification_binary: 2.3878 | Regression_cos_phi: 0.5260 | Regression_energy: 1.7422 | Regression_eta: 0.4263 | Regression_pt: 1.8349 | Regression_sin_phi: 0.4663 | LR: 2.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1834 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.2 | uss_MB: 4692.2 | child_rss_MB: 3981.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:44,321] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:25:23,568] INFO : Step 35000/50000 rank0 | Train Loss: 8.1821 | Train Loss Unweighted: 2.6372 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3414 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1266 | Regression_eta: 0.0088 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7987 | Classification_binary: 2.3414 | Regression_cos_phi: 0.5401 | Regression_energy: 1.7502 | Regression_eta: 0.4314 | Regression_pt: 1.8918 | Regression_sin_phi: 0.4283 | LR: 2.85e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1821 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3360 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5460.1 | uss_MB: 4691.9 | child_rss_MB: 3980.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:25:23,794] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:25:23,812] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 15:25:23,989] INFO : Running validation on rank0 +rank0 [2026-10-28 15:25:23,989] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:25:23,990] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:25:25,322] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 15:25:25,472] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:25:25,604] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:25:25,677] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:25:25,745] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:25:25,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:25:25,883] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:25:25,951] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:25:26,063] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:25:26,194] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:25:26,325] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:25:26,396] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:25:26,465] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:25:26,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:25:26,602] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:25:26,671] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:25:26,769] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:25:26,776] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:25:26,778] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:25:26,781] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:25:26,782] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:25:26,785] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:25:26,788] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:25:26,789] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:25:26,791] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:25:26,794] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:25:26,795] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:25:26,797] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:25:26,800] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:25:26,803] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:25:26,803] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:25:26,806] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:25:34,075] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.1821 | Valid Loss=8.2301 | Train Loss Unweighted=2.6372 | Valid Loss Unweighted=2.6745 | Stale=2 | ETA=108.4m +rank0 [2026-10-28 15:25:34,087] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:34,149] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:25:34,152] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:34,324] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:36,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:36,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:37,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:38,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:38,508] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:38,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:38,567] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:25:38,570] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:38,684] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:40,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:41,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:41,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:42,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:42,596] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:25:42,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:42,654] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:25:42,656] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:25:42,763] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:44,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:45,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:45,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:46,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:46,753] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:46,761] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:47,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:47,469] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:48,894] INFO : Plotted jets +rank0 [2026-10-28 15:25:50,002] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:50,042] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:50,264] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:50,638] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:52,612] INFO : Plotted jets +rank0 [2026-10-28 15:25:53,730] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:53,762] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 15:25:53,965] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 15:25:54,382] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:55,771] INFO : Plotted jets +rank0 [2026-10-28 15:25:56,874] INFO : Plotted jet ratio +rank0 [2026-10-28 15:26:37,351] INFO : Step 35100/50000 rank0 | Train Loss: 8.2029 | Train Loss Unweighted: 2.6480 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3447 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1322 | Regression_eta: 0.0087 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7579 | Classification_binary: 2.3447 | Regression_cos_phi: 0.4355 | Regression_energy: 1.8284 | Regression_eta: 0.4276 | Regression_pt: 1.9523 | Regression_sin_phi: 0.4566 | LR: 2.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1823 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3980.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:37,565] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:17,962] INFO : Step 35200/50000 rank0 | Train Loss: 8.5560 | Train Loss Unweighted: 2.6913 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3811 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1344 | Regression_eta: 0.0113 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7679 | Classification_binary: 2.3811 | Regression_cos_phi: 0.6226 | Regression_energy: 1.8591 | Regression_eta: 0.5575 | Regression_pt: 1.9478 | Regression_sin_phi: 0.4201 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6639s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0883 | max_backward: 0.2030 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.3340 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:27:18,178] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:57,941] INFO : Step 35300/50000 rank0 | Train Loss: 8.0533 | Train Loss Unweighted: 2.6936 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4137 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1175 | Regression_eta: 0.0085 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8140 | Classification_binary: 2.4137 | Regression_cos_phi: 0.5413 | Regression_energy: 1.6252 | Regression_eta: 0.4184 | Regression_pt: 1.7772 | Regression_sin_phi: 0.4634 | LR: 2.79e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6670s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2260 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:27:58,153] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:37,814] INFO : Step 35400/50000 rank0 | Train Loss: 7.9015 | Train Loss Unweighted: 2.6882 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4236 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1107 | Regression_eta: 0.0100 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8535 | Classification_binary: 2.4236 | Regression_cos_phi: 0.4753 | Regression_energy: 1.5314 | Regression_eta: 0.4901 | Regression_pt: 1.6157 | Regression_sin_phi: 0.5119 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0728 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3064 | max_optimizer_step: 0.1424 | max_tensorboard_logging: 0.3061 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5533.6 | uss_MB: 4766.0 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:28:38,028] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:17,548] INFO : Step 35500/50000 rank0 | Train Loss: 7.9987 | Train Loss Unweighted: 2.6968 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4107 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1241 | Regression_eta: 0.0072 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 2.4107 | Regression_cos_phi: 0.4938 | Regression_energy: 1.7156 | Regression_eta: 0.3540 | Regression_pt: 1.8250 | Regression_sin_phi: 0.3907 | LR: 2.74e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5535.6 | uss_MB: 4767.1 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:29:17,761] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:57,997] INFO : Step 35600/50000 rank0 | Train Loss: 8.7693 | Train Loss Unweighted: 2.7462 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4352 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1309 | Regression_eta: 0.0124 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8781 | Classification_binary: 2.4352 | Regression_cos_phi: 0.6227 | Regression_energy: 1.8103 | Regression_eta: 0.6089 | Regression_pt: 1.9434 | Regression_sin_phi: 0.4707 | LR: 2.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6908s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2552 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5535.6 | uss_MB: 4767.3 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:29:58,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:30:38,036] INFO : Step 35700/50000 rank0 | Train Loss: 8.5567 | Train Loss Unweighted: 2.7770 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4760 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1278 | Regression_eta: 0.0107 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.4760 | Regression_cos_phi: 0.5387 | Regression_energy: 1.7666 | Regression_eta: 0.5234 | Regression_pt: 1.8834 | Regression_sin_phi: 0.5314 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7255s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2838 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3905 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5535.6 | uss_MB: 4767.3 | child_rss_MB: 3982.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:30:38,251] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:18,051] INFO : Step 35800/50000 rank0 | Train Loss: 7.8628 | Train Loss Unweighted: 2.6751 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4103 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1111 | Regression_eta: 0.0094 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7634 | Classification_binary: 2.4103 | Regression_cos_phi: 0.5187 | Regression_energy: 1.5359 | Regression_eta: 0.4622 | Regression_pt: 1.6400 | Regression_sin_phi: 0.5323 | LR: 2.68e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8469s | Phase Times: max_pre_step_barrier: 0.1068 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0033 | max_backward: 0.3085 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5535.6 | uss_MB: 4767.4 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:31:18,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:58,634] INFO : Step 35900/50000 rank0 | Train Loss: 7.8546 | Train Loss Unweighted: 2.6254 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3479 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1187 | Regression_eta: 0.0074 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7651 | Classification_binary: 2.3479 | Regression_cos_phi: 0.4817 | Regression_energy: 1.6408 | Regression_eta: 0.3656 | Regression_pt: 1.7660 | Regression_sin_phi: 0.4874 | LR: 2.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6227s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1822 | max_loss: 0.0035 | max_backward: 0.2040 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5535.6 | uss_MB: 4767.4 | child_rss_MB: 3980.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:31:58,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:39,263] INFO : Step 36000/50000 rank0 | Train Loss: 7.7776 | Train Loss Unweighted: 2.5825 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3073 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1176 | Regression_eta: 0.0111 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7392 | Classification_binary: 2.3073 | Regression_cos_phi: 0.4414 | Regression_energy: 1.6263 | Regression_eta: 0.5440 | Regression_pt: 1.7390 | Regression_sin_phi: 0.3804 | LR: 2.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6267s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1804 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5534.8 | uss_MB: 4766.0 | child_rss_MB: 3980.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:32:39,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:39,500] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 15:32:39,684] INFO : Running validation on rank0 +rank0 [2026-10-28 15:32:39,684] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:32:39,684] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:32:40,968] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 15:32:41,112] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:32:41,242] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:32:41,315] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:32:41,384] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:32:41,454] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:32:41,523] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:32:41,592] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:32:41,707] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:32:41,838] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:32:41,970] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:32:42,040] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:32:42,109] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:32:42,178] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:32:42,246] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:32:42,316] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:32:42,417] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:32:42,423] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:32:42,426] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:32:42,429] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:32:42,430] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:32:42,432] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:32:42,435] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:32:42,436] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:32:42,438] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:32:42,441] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:32:42,444] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:32:42,445] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:32:42,448] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:32:42,449] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:32:42,451] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:32:42,454] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:32:49,785] INFO : VALIDATION | Step=36000/50000 | Train Loss=7.7776 | Valid Loss=8.1190 | Train Loss Unweighted=2.5825 | Valid Loss Unweighted=2.6484 | Stale=0 | ETA=101.2m +rank0 [2026-10-28 15:32:49,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:49,854] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:32:49,857] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:32:50,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:52,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:52,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:53,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:54,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:54,241] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:54,245] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:54,301] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:32:54,304] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:54,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:56,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:56,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:57,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:58,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:58,275] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:32:58,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:58,333] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:32:58,336] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:32:58,448] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:33:00,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:33:00,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:33:01,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:33:02,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:33:02,479] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:33:02,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:33:02,742] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:33:03,191] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:04,616] INFO : Plotted jets +rank0 [2026-10-28 15:33:05,736] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:05,769] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:33:05,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:33:06,370] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:08,311] INFO : Plotted jets +rank0 [2026-10-28 15:33:09,383] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:09,416] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 15:33:09,624] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 15:33:10,036] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:11,420] INFO : Plotted jets +rank0 [2026-10-28 15:33:12,532] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:53,255] INFO : Step 36100/50000 rank0 | Train Loss: 8.0080 | Train Loss Unweighted: 2.6886 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4015 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1235 | Regression_eta: 0.0092 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7719 | Classification_binary: 2.4015 | Regression_cos_phi: 0.4225 | Regression_energy: 1.7082 | Regression_eta: 0.4527 | Regression_pt: 1.8417 | Regression_sin_phi: 0.4095 | LR: 2.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0765 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3085 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 4012.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:33:53,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:34:33,513] INFO : Step 36200/50000 rank0 | Train Loss: 8.3417 | Train Loss Unweighted: 2.6879 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3976 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1215 | Regression_eta: 0.0129 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8017 | Classification_binary: 2.3976 | Regression_cos_phi: 0.5800 | Regression_energy: 1.6802 | Regression_eta: 0.6319 | Regression_pt: 1.8106 | Regression_sin_phi: 0.4396 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6203s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0767 | max_backward: 0.2034 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:34:33,726] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:13,717] INFO : Step 36300/50000 rank0 | Train Loss: 7.6702 | Train Loss Unweighted: 2.6941 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4334 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1107 | Regression_eta: 0.0074 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8087 | Classification_binary: 2.4334 | Regression_cos_phi: 0.4709 | Regression_energy: 1.5314 | Regression_eta: 0.3630 | Regression_pt: 1.6355 | Regression_sin_phi: 0.4273 | LR: 2.57e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7629s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1807 | max_loss: 0.0035 | max_backward: 0.2039 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 3983.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:13,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:54,359] INFO : Step 36400/50000 rank0 | Train Loss: 8.1265 | Train Loss Unweighted: 2.7082 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4306 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1160 | Regression_eta: 0.0093 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4306 | Regression_cos_phi: 0.5584 | Regression_energy: 1.6046 | Regression_eta: 0.4562 | Regression_pt: 1.7339 | Regression_sin_phi: 0.5293 | LR: 2.55e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1058 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.3079 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5610.8 | uss_MB: 4841.0 | child_rss_MB: 3982.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:54,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:36:33,851] INFO : Step 36500/50000 rank0 | Train Loss: 7.5127 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3359 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1141 | Regression_eta: 0.0079 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7155 | Classification_binary: 2.3359 | Regression_cos_phi: 0.4340 | Regression_energy: 1.5775 | Regression_eta: 0.3891 | Regression_pt: 1.7180 | Regression_sin_phi: 0.3426 | LR: 2.52e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0116 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3077 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.2905 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5611.8 | uss_MB: 4842.3 | child_rss_MB: 4009.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:36:34,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:14,223] INFO : Step 36600/50000 rank0 | Train Loss: 7.5634 | Train Loss Unweighted: 2.5979 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3412 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1082 | Regression_eta: 0.0088 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7725 | Classification_binary: 2.3412 | Regression_cos_phi: 0.4918 | Regression_energy: 1.4958 | Regression_eta: 0.4318 | Regression_pt: 1.6011 | Regression_sin_phi: 0.4293 | LR: 2.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8322s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1051 | max_loss: 0.0037 | max_backward: 0.3945 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.3485 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5611.8 | uss_MB: 4845.0 | child_rss_MB: 3980.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:37:14,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:54,482] INFO : Step 36700/50000 rank0 | Train Loss: 7.9819 | Train Loss Unweighted: 2.6507 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3656 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1222 | Regression_eta: 0.0090 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7050 | Classification_binary: 2.3656 | Regression_cos_phi: 0.4988 | Regression_energy: 1.6903 | Regression_eta: 0.4405 | Regression_pt: 1.8311 | Regression_sin_phi: 0.4507 | LR: 2.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7500s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1845 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2364 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5611.8 | uss_MB: 4843.0 | child_rss_MB: 3986.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:37:54,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:38:35,347] INFO : Step 36800/50000 rank0 | Train Loss: 8.4863 | Train Loss Unweighted: 2.6280 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3215 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1297 | Regression_eta: 0.0086 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8425 | Classification_binary: 2.3215 | Regression_cos_phi: 0.4925 | Regression_energy: 1.7940 | Regression_eta: 0.4246 | Regression_pt: 1.9379 | Regression_sin_phi: 0.6734 | LR: 2.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2756 | max_optimizer_step: 0.1394 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5611.8 | uss_MB: 4843.0 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:38:35,562] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:14,954] INFO : Step 36900/50000 rank0 | Train Loss: 8.0493 | Train Loss Unweighted: 2.5536 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2624 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1246 | Regression_eta: 0.0109 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.2624 | Regression_cos_phi: 0.5934 | Regression_energy: 1.7236 | Regression_eta: 0.5378 | Regression_pt: 1.8514 | Regression_sin_phi: 0.3476 | LR: 2.44e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7632s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2119 | max_loss: 0.0035 | max_backward: 0.2885 | max_optimizer_step: 0.1570 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5611.8 | uss_MB: 4845.1 | child_rss_MB: 3983.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:39:15,167] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:54,749] INFO : Step 37000/50000 rank0 | Train Loss: 8.3445 | Train Loss Unweighted: 2.6246 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3143 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1346 | Regression_eta: 0.0099 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7942 | Classification_binary: 2.3143 | Regression_cos_phi: 0.4666 | Regression_energy: 1.8608 | Regression_eta: 0.4876 | Regression_pt: 1.9912 | Regression_sin_phi: 0.4299 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6626s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2137 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3976 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5609.9 | uss_MB: 4841.1 | child_rss_MB: 3979.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:54,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:55,014] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 15:39:55,195] INFO : Running validation on rank0 +rank0 [2026-10-28 15:39:55,196] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:39:55,196] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:39:56,564] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 15:39:56,709] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:39:56,840] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:39:56,914] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:39:56,982] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:39:57,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:39:57,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:39:57,188] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:39:57,303] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:39:57,434] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:39:57,564] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:39:57,635] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:39:57,705] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:39:57,773] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:39:57,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:39:57,912] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:39:58,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:39:58,017] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:39:58,020] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:39:58,023] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:39:58,026] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:39:58,027] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:39:58,029] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:39:58,030] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:39:58,033] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:39:58,035] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:39:58,037] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:39:58,039] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:39:58,043] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:39:58,045] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:39:58,046] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:39:58,049] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:40:05,366] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.3445 | Valid Loss=8.0830 | Train Loss Unweighted=2.6246 | Valid Loss Unweighted=2.6450 | Stale=0 | ETA=94.0m +rank0 [2026-10-28 15:40:05,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:05,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:40:05,441] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:05,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:07,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:08,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:08,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:09,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:09,785] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:09,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:09,844] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:40:09,847] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:09,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:11,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:12,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:13,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:13,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:13,848] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:40:13,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:13,925] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:40:13,928] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:40:14,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:15,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:16,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:17,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:17,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:18,039] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:18,046] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:18,304] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:18,753] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:20,175] INFO : Plotted jets +rank0 [2026-10-28 15:40:21,883] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:21,915] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:22,144] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:22,518] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:23,898] INFO : Plotted jets +rank0 [2026-10-28 15:40:24,958] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:24,991] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 15:40:25,198] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 15:40:25,611] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:26,995] INFO : Plotted jets +rank0 [2026-10-28 15:40:28,098] INFO : Plotted jet ratio +rank0 [2026-10-28 15:41:07,678] INFO : Step 37100/50000 rank0 | Train Loss: 7.9909 | Train Loss Unweighted: 2.5852 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2988 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1223 | Regression_eta: 0.0099 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7379 | Classification_binary: 2.2988 | Regression_cos_phi: 0.4489 | Regression_energy: 1.6915 | Regression_eta: 0.4885 | Regression_pt: 1.8181 | Regression_sin_phi: 0.5071 | LR: 2.40e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5701s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0113 | max_backward: 0.2028 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5809.6 | uss_MB: 5041.8 | child_rss_MB: 3978.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:41:07,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:41:47,658] INFO : Step 37200/50000 rank0 | Train Loss: 8.2503 | Train Loss Unweighted: 2.7180 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4268 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1227 | Regression_eta: 0.0087 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8284 | Classification_binary: 2.4268 | Regression_cos_phi: 0.5316 | Regression_energy: 1.6962 | Regression_eta: 0.4253 | Regression_pt: 1.8605 | Regression_sin_phi: 0.4814 | LR: 2.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0796 | max_backward: 0.2031 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5809.6 | uss_MB: 5041.9 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:41:47,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:42:27,410] INFO : Step 37300/50000 rank0 | Train Loss: 7.3402 | Train Loss Unweighted: 2.5243 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2617 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1134 | Regression_eta: 0.0068 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7017 | Classification_binary: 2.2617 | Regression_cos_phi: 0.4013 | Regression_energy: 1.5676 | Regression_eta: 0.3328 | Regression_pt: 1.7031 | Regression_sin_phi: 0.3721 | LR: 2.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6962s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0417 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3048 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5809.6 | uss_MB: 5041.9 | child_rss_MB: 4014.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:42:27,622] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:07,076] INFO : Step 37400/50000 rank0 | Train Loss: 7.8503 | Train Loss Unweighted: 2.5392 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.2656 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1155 | Regression_eta: 0.0092 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 2.2656 | Regression_cos_phi: 0.5365 | Regression_energy: 1.5966 | Regression_eta: 0.4528 | Regression_pt: 1.6971 | Regression_sin_phi: 0.4738 | LR: 2.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0037 | max_backward: 0.2398 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.4038 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5682.2 | uss_MB: 4913.5 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:07,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:47,601] INFO : Step 37500/50000 rank0 | Train Loss: 8.0030 | Train Loss Unweighted: 2.5910 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3061 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1210 | Regression_eta: 0.0085 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7763 | Classification_binary: 2.3061 | Regression_cos_phi: 0.5130 | Regression_energy: 1.6739 | Regression_eta: 0.4166 | Regression_pt: 1.8099 | Regression_sin_phi: 0.5072 | LR: 2.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8625s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1088 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2890 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5683.2 | uss_MB: 4914.9 | child_rss_MB: 3983.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:47,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:44:28,383] INFO : Step 37600/50000 rank0 | Train Loss: 8.5621 | Train Loss Unweighted: 2.6155 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3060 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1308 | Regression_eta: 0.0140 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.3060 | Regression_cos_phi: 0.4866 | Regression_energy: 1.8084 | Regression_eta: 0.6858 | Regression_pt: 1.9351 | Regression_sin_phi: 0.4900 | LR: 2.30e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8536s | Phase Times: max_pre_step_barrier: 0.0976 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3120 | max_optimizer_step: 0.1423 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5684.2 | uss_MB: 4915.1 | child_rss_MB: 3982.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:44:28,596] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:08,795] INFO : Step 37700/50000 rank0 | Train Loss: 8.5215 | Train Loss Unweighted: 2.6826 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3794 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1288 | Regression_eta: 0.0073 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8191 | Classification_binary: 2.3794 | Regression_cos_phi: 0.7821 | Regression_energy: 1.7814 | Regression_eta: 0.3599 | Regression_pt: 1.9141 | Regression_sin_phi: 0.4854 | LR: 2.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7930s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2118 | max_loss: 0.0036 | max_backward: 0.2876 | max_optimizer_step: 0.1528 | max_tensorboard_logging: 0.3036 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5684.2 | uss_MB: 4917.3 | child_rss_MB: 3983.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:45:09,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:49,610] INFO : Step 37800/50000 rank0 | Train Loss: 8.3855 | Train Loss Unweighted: 2.7112 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1257 | Regression_eta: 0.0102 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8711 | Classification_binary: 2.4148 | Regression_cos_phi: 0.5790 | Regression_energy: 1.7387 | Regression_eta: 0.4988 | Regression_pt: 1.8590 | Regression_sin_phi: 0.4241 | LR: 2.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1641 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2169 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5684.2 | uss_MB: 4915.3 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:45:49,827] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:46:29,652] INFO : Step 37900/50000 rank0 | Train Loss: 8.3340 | Train Loss Unweighted: 2.5603 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.2606 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1273 | Regression_eta: 0.0084 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.2606 | Regression_cos_phi: 0.6635 | Regression_energy: 1.7607 | Regression_eta: 0.4132 | Regression_pt: 1.8870 | Regression_sin_phi: 0.5528 | LR: 2.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6332s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0804 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5684.2 | uss_MB: 4915.4 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:46:29,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:09,230] INFO : Step 38000/50000 rank0 | Train Loss: 8.0994 | Train Loss Unweighted: 2.5566 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.2575 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1286 | Regression_eta: 0.0073 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8421 | Classification_binary: 2.2575 | Regression_cos_phi: 0.4979 | Regression_energy: 1.7777 | Regression_eta: 0.3606 | Regression_pt: 1.9249 | Regression_sin_phi: 0.4387 | LR: 2.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0824 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3104 | max_optimizer_step: 0.1389 | max_tensorboard_logging: 0.3049 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5682.1 | uss_MB: 4913.5 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:47:09,442] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:09,461] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 15:47:09,743] INFO : Running validation on rank0 +rank0 [2026-10-28 15:47:09,743] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:47:09,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:47:10,926] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 15:47:11,069] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:47:11,200] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:47:11,273] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:47:11,342] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:47:11,411] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:47:11,480] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:47:11,550] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:47:11,663] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:47:11,793] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:47:11,925] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:47:11,996] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:47:12,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:47:12,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:47:12,201] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:47:12,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:47:12,371] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:47:12,473] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:47:12,476] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:47:12,478] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:47:12,481] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:47:12,482] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:47:12,485] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:47:12,488] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:47:12,489] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:47:12,492] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:47:12,493] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:47:12,495] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:47:12,498] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:47:12,501] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:47:12,504] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:47:12,507] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:47:19,881] INFO : VALIDATION | Step=38000/50000 | Train Loss=8.0994 | Valid Loss=8.0654 | Train Loss Unweighted=2.5566 | Valid Loss Unweighted=2.6337 | Stale=0 | ETA=86.8m +rank0 [2026-10-28 15:47:19,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:19,954] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:47:19,957] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:20,096] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:22,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:22,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:23,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:24,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:24,272] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:24,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:24,332] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:47:24,335] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:24,445] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:26,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:26,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:27,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:28,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:28,408] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:28,465] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:47:28,468] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:47:28,580] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:30,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:30,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:31,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:32,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:32,529] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:47:32,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:32,795] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:33,245] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:34,671] INFO : Plotted jets +rank0 [2026-10-28 15:47:36,370] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:36,403] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:36,633] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:37,006] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:38,388] INFO : Plotted jets +rank0 [2026-10-28 15:47:39,469] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:39,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 15:47:39,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 15:47:40,125] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:41,519] INFO : Plotted jets +rank0 [2026-10-28 15:47:42,634] INFO : Plotted jet ratio +rank0 [2026-10-28 15:48:23,373] INFO : Step 38100/50000 rank0 | Train Loss: 7.7936 | Train Loss Unweighted: 2.5981 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3114 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1244 | Regression_eta: 0.0072 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7189 | Classification_binary: 2.3114 | Regression_cos_phi: 0.4470 | Regression_energy: 1.7198 | Regression_eta: 0.3544 | Regression_pt: 1.8806 | Regression_sin_phi: 0.3615 | LR: 2.20e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1069 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.2961 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5883.1 | uss_MB: 5114.8 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:48:23,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:03,549] INFO : Step 38200/50000 rank0 | Train Loss: 7.8267 | Train Loss Unweighted: 2.5147 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.2350 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1192 | Regression_eta: 0.0112 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8067 | Classification_binary: 2.2350 | Regression_cos_phi: 0.4280 | Regression_energy: 1.6484 | Regression_eta: 0.5515 | Regression_pt: 1.7521 | Regression_sin_phi: 0.4050 | LR: 2.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1002 | max_forward: 0.0779 | max_loss: 0.0037 | max_backward: 0.3090 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5883.1 | uss_MB: 5114.9 | child_rss_MB: 4012.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:49:03,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:43,492] INFO : Step 38300/50000 rank0 | Train Loss: 7.7875 | Train Loss Unweighted: 2.6139 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3447 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1126 | Regression_eta: 0.0089 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7817 | Classification_binary: 2.3447 | Regression_cos_phi: 0.5217 | Regression_energy: 1.5564 | Regression_eta: 0.4378 | Regression_pt: 1.7102 | Regression_sin_phi: 0.4350 | LR: 2.16e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7434s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1760 | max_loss: 0.0036 | max_backward: 0.2042 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5883.1 | uss_MB: 5114.9 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:49:43,704] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:50:23,323] INFO : Step 38400/50000 rank0 | Train Loss: 8.2408 | Train Loss Unweighted: 2.6439 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3527 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1233 | Regression_eta: 0.0092 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7712 | Classification_binary: 2.3527 | Regression_cos_phi: 0.6665 | Regression_energy: 1.7053 | Regression_eta: 0.4509 | Regression_pt: 1.8407 | Regression_sin_phi: 0.4536 | LR: 2.14e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7979s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2971 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.4229 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 5754.2 | uss_MB: 4985.4 | child_rss_MB: 3986.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:50:23,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:03,087] INFO : Step 38500/50000 rank0 | Train Loss: 7.8348 | Train Loss Unweighted: 2.6076 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3319 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1173 | Regression_eta: 0.0094 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7822 | Classification_binary: 2.3319 | Regression_cos_phi: 0.5312 | Regression_energy: 1.6222 | Regression_eta: 0.4611 | Regression_pt: 1.7457 | Regression_sin_phi: 0.3604 | LR: 2.12e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6750s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0148 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3104 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5755.2 | uss_MB: 4986.2 | child_rss_MB: 4014.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:51:03,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:43,196] INFO : Step 38600/50000 rank0 | Train Loss: 8.2647 | Train Loss Unweighted: 2.7121 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4227 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1217 | Regression_eta: 0.0097 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7787 | Classification_binary: 2.4227 | Regression_cos_phi: 0.5987 | Regression_energy: 1.6834 | Regression_eta: 0.4747 | Regression_pt: 1.8383 | Regression_sin_phi: 0.4682 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1478 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.3091 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5755.2 | uss_MB: 4986.3 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:51:43,408] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:52:23,599] INFO : Step 38700/50000 rank0 | Train Loss: 8.8647 | Train Loss Unweighted: 2.6434 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2973 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1509 | Regression_eta: 0.0087 | Regression_pt: 0.1521 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7761 | Classification_binary: 2.2973 | Regression_cos_phi: 0.5739 | Regression_energy: 2.0864 | Regression_eta: 0.4292 | Regression_pt: 2.2879 | Regression_sin_phi: 0.4138 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2138 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3925 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5755.2 | uss_MB: 4986.4 | child_rss_MB: 3980.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:52:23,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:03,828] INFO : Step 38800/50000 rank0 | Train Loss: 7.6245 | Train Loss Unweighted: 2.5643 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3039 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1108 | Regression_eta: 0.0084 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7351 | Classification_binary: 2.3039 | Regression_cos_phi: 0.5772 | Regression_energy: 1.5319 | Regression_eta: 0.4137 | Regression_pt: 1.6097 | Regression_sin_phi: 0.4529 | LR: 2.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3109 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3917 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5755.2 | uss_MB: 4986.6 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:53:04,040] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:43,808] INFO : Step 38900/50000 rank0 | Train Loss: 8.1884 | Train Loss Unweighted: 2.6882 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3896 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1280 | Regression_eta: 0.0084 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7737 | Classification_binary: 2.3896 | Regression_cos_phi: 0.4837 | Regression_energy: 1.7696 | Regression_eta: 0.4134 | Regression_pt: 1.9453 | Regression_sin_phi: 0.4132 | LR: 2.05e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7467s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3057 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.3753 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5755.2 | uss_MB: 4986.6 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:53:44,021] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:23,982] INFO : Step 39000/50000 rank0 | Train Loss: 7.7432 | Train Loss Unweighted: 2.5425 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.2664 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1170 | Regression_eta: 0.0090 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8016 | Classification_binary: 2.2664 | Regression_cos_phi: 0.4368 | Regression_energy: 1.6173 | Regression_eta: 0.4445 | Regression_pt: 1.7629 | Regression_sin_phi: 0.4136 | LR: 2.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0039 | max_backward: 0.2033 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3279 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5753.8 | uss_MB: 4985.2 | child_rss_MB: 3985.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:54:24,196] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:24,218] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 15:54:24,396] INFO : Running validation on rank0 +rank0 [2026-10-28 15:54:24,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:54:24,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:54:25,675] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 15:54:25,821] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:54:25,951] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:54:26,025] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:54:26,093] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:54:26,160] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:54:26,230] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:54:26,299] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:54:26,413] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:54:26,544] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:54:26,675] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:54:26,745] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:54:26,814] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:54:26,884] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:54:26,952] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:54:27,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:54:27,125] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:54:27,132] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:54:27,135] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:54:27,136] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:54:27,139] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:54:27,142] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:54:27,144] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:54:27,147] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:54:27,150] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:54:27,153] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:54:27,154] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:54:27,157] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:54:27,159] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:54:27,160] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:54:27,163] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:54:27,166] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:54:34,493] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.7432 | Valid Loss=8.1049 | Train Loss Unweighted=2.5425 | Valid Loss Unweighted=2.6448 | Stale=1 | ETA=79.5m +rank0 [2026-10-28 15:54:34,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:34,596] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:54:34,600] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:34,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:36,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:37,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:38,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:38,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:39,008] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:54:39,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:39,066] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:54:39,069] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:39,184] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:41,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:41,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:42,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:42,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:43,100] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:54:43,103] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:43,157] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:54:43,160] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:54:43,278] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:44,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:45,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:46,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:47,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:47,191] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:54:47,199] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:47,454] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:47,901] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:49,335] INFO : Plotted jets +rank0 [2026-10-28 15:54:50,454] INFO : Plotted jet ratio +rank0 [2026-10-28 15:54:50,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:50,712] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:51,088] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:53,037] INFO : Plotted jets +rank0 [2026-10-28 15:54:54,106] INFO : Plotted jet ratio +rank0 [2026-10-28 15:54:54,139] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 15:54:54,340] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 15:54:54,759] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:56,150] INFO : Plotted jets +rank0 [2026-10-28 15:54:57,259] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:37,171] INFO : Step 39100/50000 rank0 | Train Loss: 7.7673 | Train Loss Unweighted: 2.5947 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3245 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1154 | Regression_eta: 0.0090 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.3245 | Regression_cos_phi: 0.5179 | Regression_energy: 1.5958 | Regression_eta: 0.4441 | Regression_pt: 1.6847 | Regression_sin_phi: 0.4156 | LR: 2.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0235 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3073 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 4015.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:55:37,386] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:16,671] INFO : Step 39200/50000 rank0 | Train Loss: 7.9284 | Train Loss Unweighted: 2.5878 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3093 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1173 | Regression_eta: 0.0095 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.3093 | Regression_cos_phi: 0.5089 | Regression_energy: 1.6225 | Regression_eta: 0.4690 | Regression_pt: 1.7625 | Regression_sin_phi: 0.4915 | LR: 2.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7801s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0248 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3112 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:56:16,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:56,424] INFO : Step 39300/50000 rank0 | Train Loss: 8.3931 | Train Loss Unweighted: 2.6486 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3367 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1340 | Regression_eta: 0.0090 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7401 | Classification_binary: 2.3367 | Regression_cos_phi: 0.5352 | Regression_energy: 1.8532 | Regression_eta: 0.4399 | Regression_pt: 2.0361 | Regression_sin_phi: 0.4520 | LR: 1.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7440s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1790 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 3985.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:56:56,638] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:35,963] INFO : Step 39400/50000 rank0 | Train Loss: 8.3117 | Train Loss Unweighted: 2.6444 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3441 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1273 | Regression_eta: 0.0092 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7322 | Classification_binary: 2.3441 | Regression_cos_phi: 0.5650 | Regression_energy: 1.7607 | Regression_eta: 0.4515 | Regression_pt: 1.9358 | Regression_sin_phi: 0.5224 | LR: 1.96e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6803s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0157 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.3073 | max_optimizer_step: 0.1426 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 4011.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:57:36,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:15,383] INFO : Step 39500/50000 rank0 | Train Loss: 7.8927 | Train Loss Unweighted: 2.6213 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3375 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1217 | Regression_eta: 0.0073 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.3375 | Regression_cos_phi: 0.4761 | Regression_energy: 1.6826 | Regression_eta: 0.3567 | Regression_pt: 1.8265 | Regression_sin_phi: 0.4152 | LR: 1.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 5826.9 | uss_MB: 5060.2 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:58:15,597] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:55,527] INFO : Step 39600/50000 rank0 | Train Loss: 7.5867 | Train Loss Unweighted: 2.5958 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3279 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1136 | Regression_eta: 0.0082 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8227 | Classification_binary: 2.3279 | Regression_cos_phi: 0.3909 | Regression_energy: 1.5704 | Regression_eta: 0.4026 | Regression_pt: 1.7239 | Regression_sin_phi: 0.3481 | LR: 1.93e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7334s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3016 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5826.9 | uss_MB: 5060.4 | child_rss_MB: 3986.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:58:55,736] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:59:35,507] INFO : Step 39700/50000 rank0 | Train Loss: 7.6111 | Train Loss Unweighted: 2.6369 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3809 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1063 | Regression_eta: 0.0095 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7968 | Classification_binary: 2.3809 | Regression_cos_phi: 0.4977 | Regression_energy: 1.4698 | Regression_eta: 0.4687 | Regression_pt: 1.6071 | Regression_sin_phi: 0.3902 | LR: 1.91e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7439s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3097 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5826.9 | uss_MB: 5060.5 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:59:35,722] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:16,036] INFO : Step 39800/50000 rank0 | Train Loss: 8.4719 | Train Loss Unweighted: 2.7243 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4189 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1300 | Regression_eta: 0.0076 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.4189 | Regression_cos_phi: 0.6634 | Regression_energy: 1.7972 | Regression_eta: 0.3712 | Regression_pt: 1.9708 | Regression_sin_phi: 0.4577 | LR: 1.89e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1357 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5826.9 | uss_MB: 5060.6 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:16,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:56,451] INFO : Step 39900/50000 rank0 | Train Loss: 8.0349 | Train Loss Unweighted: 2.6589 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3860 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1144 | Regression_eta: 0.0107 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7612 | Classification_binary: 2.3860 | Regression_cos_phi: 0.5875 | Regression_energy: 1.5817 | Regression_eta: 0.5243 | Regression_pt: 1.6853 | Regression_sin_phi: 0.5089 | LR: 1.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3039 | max_optimizer_step: 0.1403 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5826.9 | uss_MB: 5060.7 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:56,663] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:36,356] INFO : Step 40000/50000 rank0 | Train Loss: 8.4132 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.3896 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1267 | Regression_eta: 0.0101 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.3896 | Regression_cos_phi: 0.5595 | Regression_energy: 1.7515 | Regression_eta: 0.4964 | Regression_pt: 1.9134 | Regression_sin_phi: 0.4352 | LR: 1.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7614s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3097 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.9 | uss_MB: 5060.7 | child_rss_MB: 3981.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:01:36,567] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:36,585] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 16:01:36,865] INFO : Running validation on rank0 +rank0 [2026-10-28 16:01:36,865] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:01:36,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:01:38,059] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 16:01:38,202] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:01:38,334] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:01:38,407] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:01:38,476] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:01:38,544] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:01:38,613] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:01:38,682] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:01:38,796] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:01:38,927] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:01:39,059] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:01:39,131] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:01:39,200] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:01:39,268] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:01:39,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:01:39,406] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:01:39,506] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:01:39,609] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:01:39,612] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:01:39,615] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:01:39,617] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:01:39,619] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:01:39,622] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:01:39,624] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:01:39,625] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:01:39,628] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:01:39,629] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:01:39,632] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:01:39,635] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:01:39,637] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:01:39,638] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:01:39,641] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:01:46,951] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.4132 | Valid Loss=8.0601 | Train Loss Unweighted=2.6903 | Valid Loss Unweighted=2.6280 | Stale=0 | ETA=72.3m +rank0 [2026-10-28 16:01:46,960] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:47,020] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:01:47,024] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:01:47,198] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:49,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:49,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:50,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:51,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:51,454] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:51,458] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:51,513] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:01:51,516] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:01:51,628] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:53,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:54,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:54,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:55,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:55,621] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:55,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:55,678] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:01:55,681] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:01:55,797] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:57,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:58,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:58,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:59,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:59,760] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:59,768] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:02:00,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:02:00,469] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:01,899] INFO : Plotted jets +rank0 [2026-10-28 16:02:03,590] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:03,621] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:03,850] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:04,224] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:05,608] INFO : Plotted jets +rank0 [2026-10-28 16:02:06,685] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:06,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 16:02:06,922] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 16:02:07,335] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:08,726] INFO : Plotted jets +rank0 [2026-10-28 16:02:09,836] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:49,847] INFO : Step 40100/50000 rank0 | Train Loss: 7.9511 | Train Loss Unweighted: 2.5532 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.2735 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1186 | Regression_eta: 0.0092 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7573 | Classification_binary: 2.2735 | Regression_cos_phi: 0.6014 | Regression_energy: 1.6405 | Regression_eta: 0.4498 | Regression_pt: 1.7511 | Regression_sin_phi: 0.4775 | LR: 1.84e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7737s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1843 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2497 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6028.5 | uss_MB: 5264.4 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:02:50,062] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:03:30,075] INFO : Step 40200/50000 rank0 | Train Loss: 7.8004 | Train Loss Unweighted: 2.5651 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2961 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1124 | Regression_eta: 0.0093 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7838 | Classification_binary: 2.2961 | Regression_cos_phi: 0.5573 | Regression_energy: 1.5542 | Regression_eta: 0.4546 | Regression_pt: 1.6863 | Regression_sin_phi: 0.4681 | LR: 1.83e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8227s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0550 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1426 | max_tensorboard_logging: 0.2792 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6028.5 | uss_MB: 5264.5 | child_rss_MB: 3985.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:03:30,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:10,914] INFO : Step 40300/50000 rank0 | Train Loss: 7.8063 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3289 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1159 | Regression_eta: 0.0087 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7828 | Classification_binary: 2.3289 | Regression_cos_phi: 0.5544 | Regression_energy: 1.6025 | Regression_eta: 0.4280 | Regression_pt: 1.7181 | Regression_sin_phi: 0.3916 | LR: 1.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0067 | max_backward: 0.2038 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.2828 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6028.5 | uss_MB: 5264.5 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:04:11,129] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:50,339] INFO : Step 40400/50000 rank0 | Train Loss: 7.9823 | Train Loss Unweighted: 2.5962 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3061 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1263 | Regression_eta: 0.0088 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7754 | Classification_binary: 2.3061 | Regression_cos_phi: 0.4104 | Regression_energy: 1.7466 | Regression_eta: 0.4345 | Regression_pt: 1.8353 | Regression_sin_phi: 0.4740 | LR: 1.79e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7697s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1046 | max_loss: 0.0744 | max_backward: 0.2885 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6028.5 | uss_MB: 5266.6 | child_rss_MB: 3982.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:04:50,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:05:30,766] INFO : Step 40500/50000 rank0 | Train Loss: 8.3412 | Train Loss Unweighted: 2.6182 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3144 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1295 | Regression_eta: 0.0105 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8055 | Classification_binary: 2.3144 | Regression_cos_phi: 0.5977 | Regression_energy: 1.7912 | Regression_eta: 0.5141 | Regression_pt: 1.9396 | Regression_sin_phi: 0.3786 | LR: 1.78e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7701s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0020 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3093 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 5897.9 | uss_MB: 5133.5 | child_rss_MB: 3983.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:05:30,978] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:11,347] INFO : Step 40600/50000 rank0 | Train Loss: 7.8341 | Train Loss Unweighted: 2.6014 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3248 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1174 | Regression_eta: 0.0081 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8461 | Classification_binary: 2.3248 | Regression_cos_phi: 0.4745 | Regression_energy: 1.6233 | Regression_eta: 0.3997 | Regression_pt: 1.7530 | Regression_sin_phi: 0.4127 | LR: 1.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7665s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1041 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3099 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5897.9 | uss_MB: 5133.5 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:06:11,560] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:51,240] INFO : Step 40700/50000 rank0 | Train Loss: 7.4236 | Train Loss Unweighted: 2.5730 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3109 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1129 | Regression_eta: 0.0059 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.3109 | Regression_cos_phi: 0.4629 | Regression_energy: 1.5613 | Regression_eta: 0.2876 | Regression_pt: 1.6888 | Regression_sin_phi: 0.3700 | LR: 1.75e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6301s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0037 | max_backward: 0.2038 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.2804 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3979.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:06:51,455] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:07:31,033] INFO : Step 40800/50000 rank0 | Train Loss: 8.0254 | Train Loss Unweighted: 2.6312 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.3607 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1133 | Regression_eta: 0.0100 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8528 | Classification_binary: 2.3607 | Regression_cos_phi: 0.5663 | Regression_energy: 1.5666 | Regression_eta: 0.4900 | Regression_pt: 1.6353 | Regression_sin_phi: 0.5538 | LR: 1.73e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0964 | max_backward: 0.2032 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:07:31,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:11,386] INFO : Step 40900/50000 rank0 | Train Loss: 7.8040 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3293 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1182 | Regression_eta: 0.0089 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7306 | Classification_binary: 2.3293 | Regression_cos_phi: 0.5172 | Regression_energy: 1.6348 | Regression_eta: 0.4397 | Regression_pt: 1.6796 | Regression_sin_phi: 0.4728 | LR: 1.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6957s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0037 | max_backward: 0.2709 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3981.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:08:11,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:52,069] INFO : Step 41000/50000 rank0 | Train Loss: 8.2526 | Train Loss Unweighted: 2.7058 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4090 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1263 | Regression_eta: 0.0091 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7721 | Classification_binary: 2.4090 | Regression_cos_phi: 0.5069 | Regression_energy: 1.7460 | Regression_eta: 0.4495 | Regression_pt: 1.9187 | Regression_sin_phi: 0.4504 | LR: 1.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3105 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5897.9 | uss_MB: 5134.0 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:08:52,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:52,386] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 16:08:52,562] INFO : Running validation on rank0 +rank0 [2026-10-28 16:08:52,562] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:08:52,562] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:08:53,843] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 16:08:53,986] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:08:54,118] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:08:54,191] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:08:54,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:08:54,329] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:08:54,398] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:08:54,466] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:08:54,582] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:08:54,714] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:08:54,845] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:08:54,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:08:54,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:08:55,053] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:08:55,122] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:08:55,191] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:08:55,291] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:08:55,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:08:55,300] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:08:55,303] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:08:55,303] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:08:55,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:08:55,308] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:08:55,310] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:08:55,311] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:08:55,314] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:08:55,317] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:08:55,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:08:55,320] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:08:55,323] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:08:55,323] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:08:55,327] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:09:02,639] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.2526 | Valid Loss=8.0337 | Train Loss Unweighted=2.7058 | Valid Loss Unweighted=2.6288 | Stale=0 | ETA=65.1m +rank0 [2026-10-28 16:09:02,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:02,711] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:09:02,715] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:02,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:04,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:05,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:06,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:06,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:07,007] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:07,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:07,068] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:09:07,071] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:07,185] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:08,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:09,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:10,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:10,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:11,165] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:11,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:11,223] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:09:11,226] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:09:11,343] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:13,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:13,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:14,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:15,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:15,281] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:09:15,289] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:15,543] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:15,992] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:17,418] INFO : Plotted jets +rank0 [2026-10-28 16:09:18,545] INFO : Plotted jet ratio +rank0 [2026-10-28 16:09:18,577] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:18,802] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:19,174] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:21,187] INFO : Plotted jets +rank0 [2026-10-28 16:09:22,256] INFO : Plotted jet ratio +rank0 [2026-10-28 16:09:22,288] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 16:09:22,493] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 16:09:22,908] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:24,291] INFO : Plotted jets +rank0 [2026-10-28 16:09:25,397] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:06,437] INFO : Step 41100/50000 rank0 | Train Loss: 8.6334 | Train Loss Unweighted: 2.6600 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3525 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1300 | Regression_eta: 0.0101 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8333 | Classification_binary: 2.3525 | Regression_cos_phi: 0.7147 | Regression_energy: 1.7971 | Regression_eta: 0.4983 | Regression_pt: 1.9221 | Regression_sin_phi: 0.5154 | LR: 1.69e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7252s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.2466 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6089.4 | uss_MB: 5322.7 | child_rss_MB: 3976.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:10:06,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:10:46,396] INFO : Step 41200/50000 rank0 | Train Loss: 8.3401 | Train Loss Unweighted: 2.6864 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3950 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1246 | Regression_eta: 0.0112 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7746 | Classification_binary: 2.3950 | Regression_cos_phi: 0.5924 | Regression_energy: 1.7235 | Regression_eta: 0.5495 | Regression_pt: 1.7939 | Regression_sin_phi: 0.5113 | LR: 1.67e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5698s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6089.4 | uss_MB: 5322.7 | child_rss_MB: 3975.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:10:46,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:11:26,285] INFO : Step 41300/50000 rank0 | Train Loss: 7.1674 | Train Loss Unweighted: 2.4152 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.1597 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1092 | Regression_eta: 0.0061 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7066 | Classification_binary: 2.1597 | Regression_cos_phi: 0.4536 | Regression_energy: 1.5102 | Regression_eta: 0.3020 | Regression_pt: 1.6500 | Regression_sin_phi: 0.3854 | LR: 1.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0704 | max_forward: 0.1051 | max_loss: 0.0037 | max_backward: 0.3941 | max_optimizer_step: 0.1451 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6089.4 | uss_MB: 5324.8 | child_rss_MB: 3977.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:11:26,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:06,266] INFO : Step 41400/50000 rank0 | Train Loss: 8.2236 | Train Loss Unweighted: 2.6999 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4138 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1199 | Regression_eta: 0.0106 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7837 | Classification_binary: 2.4138 | Regression_cos_phi: 0.5174 | Regression_energy: 1.6579 | Regression_eta: 0.5231 | Regression_pt: 1.8049 | Regression_sin_phi: 0.5228 | LR: 1.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2075 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6089.4 | uss_MB: 5322.8 | child_rss_MB: 3979.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:12:06,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:46,717] INFO : Step 41500/50000 rank0 | Train Loss: 8.1958 | Train Loss Unweighted: 2.5559 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.2544 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1299 | Regression_eta: 0.0093 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.2544 | Regression_cos_phi: 0.4581 | Regression_energy: 1.7958 | Regression_eta: 0.4574 | Regression_pt: 1.9080 | Regression_sin_phi: 0.4545 | LR: 1.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9596s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1900 | max_loss: 0.0035 | max_backward: 0.3938 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5212.0 | child_rss_MB: 3979.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 16:12:46,930] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:13:27,100] INFO : Step 41600/50000 rank0 | Train Loss: 8.0937 | Train Loss Unweighted: 2.5803 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1264 | Regression_eta: 0.0085 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7820 | Classification_binary: 2.2874 | Regression_cos_phi: 0.5610 | Regression_energy: 1.7474 | Regression_eta: 0.4189 | Regression_pt: 1.8532 | Regression_sin_phi: 0.4437 | LR: 1.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7819s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1804 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5978.1 | uss_MB: 5210.4 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:13:27,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:07,679] INFO : Step 41700/50000 rank0 | Train Loss: 7.8485 | Train Loss Unweighted: 2.6543 | Train Loss Components: Classification: 0.0157 | Classification_binary: 2.3789 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1180 | Regression_eta: 0.0074 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6587 | Classification_binary: 2.3789 | Regression_cos_phi: 0.5719 | Regression_energy: 1.6317 | Regression_eta: 0.3617 | Regression_pt: 1.7659 | Regression_sin_phi: 0.4797 | LR: 1.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7465s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2042 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.2826 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5978.1 | uss_MB: 5210.5 | child_rss_MB: 3977.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:14:07,893] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:47,863] INFO : Step 41800/50000 rank0 | Train Loss: 8.0963 | Train Loss Unweighted: 2.6075 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3219 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1204 | Regression_eta: 0.0092 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 2.3219 | Regression_cos_phi: 0.5933 | Regression_energy: 1.6653 | Regression_eta: 0.4503 | Regression_pt: 1.8030 | Regression_sin_phi: 0.4578 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0206 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3103 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5210.6 | child_rss_MB: 3978.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:14:48,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:15:28,624] INFO : Step 41900/50000 rank0 | Train Loss: 8.3514 | Train Loss Unweighted: 2.6263 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3235 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1295 | Regression_eta: 0.0103 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7850 | Classification_binary: 2.3235 | Regression_cos_phi: 0.5657 | Regression_energy: 1.7912 | Regression_eta: 0.5052 | Regression_pt: 1.9211 | Regression_sin_phi: 0.4597 | LR: 1.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1788 | max_loss: 0.0037 | max_backward: 0.2040 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5210.7 | child_rss_MB: 3979.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:15:28,840] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:08,707] INFO : Step 42000/50000 rank0 | Train Loss: 8.2783 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4018 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1365 | Regression_eta: 0.0070 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7666 | Classification_binary: 2.4018 | Regression_cos_phi: 0.4328 | Regression_energy: 1.8872 | Regression_eta: 0.3416 | Regression_pt: 2.0377 | Regression_sin_phi: 0.4105 | LR: 1.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7029s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1060 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 0.3374 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5978.1 | uss_MB: 5210.6 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:16:08,922] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:08,943] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 16:16:09,116] INFO : Running validation on rank0 +rank0 [2026-10-28 16:16:09,116] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:16:09,116] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:16:10,409] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 16:16:10,553] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:16:10,684] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:16:10,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:16:10,829] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:16:10,898] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:16:10,967] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:16:11,036] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:16:11,152] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:16:11,282] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:16:11,414] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:16:11,485] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:16:11,555] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:16:11,624] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:16:11,693] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:16:11,762] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:16:11,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:16:11,870] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:16:11,873] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:16:11,874] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:16:11,876] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:16:11,879] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:16:11,881] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:16:11,883] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:16:11,884] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:16:11,886] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:16:11,889] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:16:11,890] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:16:11,892] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:16:11,895] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:16:11,896] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:16:11,898] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:16:19,475] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.2783 | Valid Loss=8.0147 | Train Loss Unweighted=2.7126 | Valid Loss Unweighted=2.6181 | Stale=0 | ETA=57.9m +rank0 [2026-10-28 16:16:19,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:19,545] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:16:19,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:19,724] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:21,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:22,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:23,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:23,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:24,177] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:16:24,180] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:24,237] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:16:24,240] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:24,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:26,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:26,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:27,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:28,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:28,390] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:16:28,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:28,446] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:16:28,449] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:16:28,565] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:30,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:31,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:31,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:32,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:32,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:16:32,485] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:32,740] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:33,191] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:34,622] INFO : Plotted jets +rank0 [2026-10-28 16:16:35,738] INFO : Plotted jet ratio +rank0 [2026-10-28 16:16:35,771] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:35,998] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:36,374] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:38,334] INFO : Plotted jets +rank0 [2026-10-28 16:16:39,402] INFO : Plotted jet ratio +rank0 [2026-10-28 16:16:39,435] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 16:16:39,639] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 16:16:40,056] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:41,448] INFO : Plotted jets +rank0 [2026-10-28 16:16:42,557] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:23,707] INFO : Step 42100/50000 rank0 | Train Loss: 8.4585 | Train Loss Unweighted: 2.6235 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3224 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1271 | Regression_eta: 0.0112 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8197 | Classification_binary: 2.3224 | Regression_cos_phi: 0.6258 | Regression_energy: 1.7580 | Regression_eta: 0.5521 | Regression_pt: 1.8821 | Regression_sin_phi: 0.4985 | LR: 1.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7941s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.2950 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6164.5 | uss_MB: 5398.1 | child_rss_MB: 3979.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:17:23,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:04,197] INFO : Step 42200/50000 rank0 | Train Loss: 7.4983 | Train Loss Unweighted: 2.5543 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2904 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1129 | Regression_eta: 0.0085 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7000 | Classification_binary: 2.2904 | Regression_cos_phi: 0.4133 | Regression_energy: 1.5609 | Regression_eta: 0.4188 | Regression_pt: 1.6872 | Regression_sin_phi: 0.4277 | LR: 1.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0649 | max_backward: 0.2036 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6164.5 | uss_MB: 5398.2 | child_rss_MB: 3979.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:18:04,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:10,653] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 16:18:10,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 16:18:44,430] INFO : Step 42300/50000 rank0 | Train Loss: 7.5322 | Train Loss Unweighted: 2.5345 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2787 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1080 | Regression_eta: 0.0102 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7349 | Classification_binary: 2.2787 | Regression_cos_phi: 0.5315 | Regression_energy: 1.4937 | Regression_eta: 0.4997 | Regression_pt: 1.5769 | Regression_sin_phi: 0.4168 | LR: 1.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6926s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0861 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.3095 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6203.6 | uss_MB: 5436.1 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:18:44,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:19:24,650] INFO : Step 42400/50000 rank0 | Train Loss: 7.6167 | Train Loss Unweighted: 2.5575 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.2967 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1086 | Regression_eta: 0.0081 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8257 | Classification_binary: 2.2967 | Regression_cos_phi: 0.4861 | Regression_energy: 1.5021 | Regression_eta: 0.3979 | Regression_pt: 1.6373 | Regression_sin_phi: 0.4707 | LR: 1.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9127s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0818 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1468 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 6203.6 | uss_MB: 5436.1 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:19:24,864] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:04,739] INFO : Step 42500/50000 rank0 | Train Loss: 7.6569 | Train Loss Unweighted: 2.5782 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3123 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1143 | Regression_eta: 0.0080 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7830 | Classification_binary: 2.3123 | Regression_cos_phi: 0.5299 | Regression_energy: 1.5801 | Regression_eta: 0.3923 | Regression_pt: 1.6529 | Regression_sin_phi: 0.4063 | LR: 1.49e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7544s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2821 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6049.6 | uss_MB: 5280.3 | child_rss_MB: 3980.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:20:04,951] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:44,696] INFO : Step 42600/50000 rank0 | Train Loss: 7.1602 | Train Loss Unweighted: 2.4476 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2110 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0980 | Regression_eta: 0.0081 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6828 | Classification_binary: 2.2110 | Regression_cos_phi: 0.5718 | Regression_energy: 1.3551 | Regression_eta: 0.3982 | Regression_pt: 1.4630 | Regression_sin_phi: 0.4783 | LR: 1.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6943s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2087 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3945 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 6049.6 | uss_MB: 5280.6 | child_rss_MB: 3979.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:20:44,908] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:24,541] INFO : Step 42700/50000 rank0 | Train Loss: 7.9446 | Train Loss Unweighted: 2.5983 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3126 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1228 | Regression_eta: 0.0097 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.3126 | Regression_cos_phi: 0.5042 | Regression_energy: 1.6975 | Regression_eta: 0.4745 | Regression_pt: 1.8236 | Regression_sin_phi: 0.4010 | LR: 1.47e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0127 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3063 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 6049.6 | uss_MB: 5280.7 | child_rss_MB: 3976.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:21:24,754] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:04,726] INFO : Step 42800/50000 rank0 | Train Loss: 7.1961 | Train Loss Unweighted: 2.5301 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.2812 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1060 | Regression_eta: 0.0077 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7433 | Classification_binary: 2.2812 | Regression_cos_phi: 0.3818 | Regression_energy: 1.4661 | Regression_eta: 0.3774 | Regression_pt: 1.5873 | Regression_sin_phi: 0.3591 | LR: 1.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7325s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1507 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 6049.6 | uss_MB: 5280.7 | child_rss_MB: 3977.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:22:04,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:45,226] INFO : Step 42900/50000 rank0 | Train Loss: 7.8925 | Train Loss Unweighted: 2.5584 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2783 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1190 | Regression_eta: 0.0082 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7054 | Classification_binary: 2.2783 | Regression_cos_phi: 0.5372 | Regression_energy: 1.6461 | Regression_eta: 0.4022 | Regression_pt: 1.7826 | Regression_sin_phi: 0.5406 | LR: 1.44e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.8347s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3489 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6049.6 | uss_MB: 5283.2 | child_rss_MB: 3980.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 16:22:45,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:25,811] INFO : Step 43000/50000 rank0 | Train Loss: 8.3373 | Train Loss Unweighted: 2.5881 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.2820 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1319 | Regression_eta: 0.0102 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.2820 | Regression_cos_phi: 0.5421 | Regression_energy: 1.8237 | Regression_eta: 0.5008 | Regression_pt: 1.9378 | Regression_sin_phi: 0.4444 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8501s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0596 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3048 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.4010 | max_tensorboard_flush: 0.0034 | Memory: rss_MB: 6049.6 | uss_MB: 5281.0 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:23:26,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:26,042] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 16:23:26,323] INFO : Running validation on rank0 +rank0 [2026-10-28 16:23:26,323] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:23:26,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:23:27,667] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 16:23:27,812] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:23:27,943] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:23:28,017] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:23:28,085] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:23:28,154] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:23:28,222] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:23:28,291] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:23:28,409] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:23:28,540] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:23:28,670] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:23:28,741] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:23:28,810] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:23:28,878] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:23:28,947] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:23:29,016] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:23:29,119] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:23:29,126] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:23:29,129] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:23:29,132] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:23:29,133] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:23:29,135] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:23:29,138] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:23:29,139] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:23:29,141] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:23:29,144] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:23:29,145] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:23:29,148] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:23:29,151] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:23:29,154] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:23:29,157] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:23:29,160] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:23:36,523] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.3373 | Valid Loss=7.9925 | Train Loss Unweighted=2.5881 | Valid Loss Unweighted=2.6155 | Stale=0 | ETA=50.6m +rank0 [2026-10-28 16:23:36,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:36,620] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:23:36,624] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:36,807] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:38,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:39,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:40,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:40,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:41,044] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:23:41,047] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:41,116] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:23:41,119] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:41,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:43,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:43,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:44,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:44,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:45,157] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:23:45,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:45,216] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:23:45,219] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:23:45,335] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:47,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:47,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:48,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:49,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:49,199] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:23:49,207] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:49,465] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:49,915] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:51,339] INFO : Plotted jets +rank0 [2026-10-28 16:23:53,069] INFO : Plotted jet ratio +rank0 [2026-10-28 16:23:53,104] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:53,330] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:53,703] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:55,081] INFO : Plotted jets +rank0 [2026-10-28 16:23:56,147] INFO : Plotted jet ratio +rank0 [2026-10-28 16:23:56,182] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 16:23:56,388] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 16:23:56,802] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:58,187] INFO : Plotted jets +rank0 [2026-10-28 16:23:59,293] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:39,079] INFO : Step 43100/50000 rank0 | Train Loss: 8.0236 | Train Loss Unweighted: 2.5486 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.2615 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1212 | Regression_eta: 0.0108 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7994 | Classification_binary: 2.2615 | Regression_cos_phi: 0.5191 | Regression_energy: 1.6762 | Regression_eta: 0.5304 | Regression_pt: 1.8179 | Regression_sin_phi: 0.4191 | LR: 1.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7511s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2444 | max_optimizer_step: 0.1479 | max_tensorboard_logging: 0.3908 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3646.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:24:39,292] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:25:18,579] INFO : Step 43200/50000 rank0 | Train Loss: 8.0705 | Train Loss Unweighted: 2.6043 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3090 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1276 | Regression_eta: 0.0075 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8223 | Classification_binary: 2.3090 | Regression_cos_phi: 0.5016 | Regression_energy: 1.7650 | Regression_eta: 0.3686 | Regression_pt: 1.8933 | Regression_sin_phi: 0.4108 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1780 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3982.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:25:18,792] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:25:59,093] INFO : Step 43300/50000 rank0 | Train Loss: 7.9324 | Train Loss Unweighted: 2.5775 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2880 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1242 | Regression_eta: 0.0080 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7544 | Classification_binary: 2.2880 | Regression_cos_phi: 0.4443 | Regression_energy: 1.7178 | Regression_eta: 0.3907 | Regression_pt: 1.8741 | Regression_sin_phi: 0.4632 | LR: 1.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7861s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2186 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.4285 | max_tensorboard_flush: 0.0143 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:25:59,307] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:39,097] INFO : Step 43400/50000 rank0 | Train Loss: 7.9300 | Train Loss Unweighted: 2.5945 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.3156 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1178 | Regression_eta: 0.0085 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8104 | Classification_binary: 2.3156 | Regression_cos_phi: 0.4990 | Regression_energy: 1.6297 | Regression_eta: 0.4179 | Regression_pt: 1.7613 | Regression_sin_phi: 0.4961 | LR: 1.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6722s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0829 | max_backward: 0.2031 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6240.6 | uss_MB: 5474.8 | child_rss_MB: 3981.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:26:39,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:18,958] INFO : Step 43500/50000 rank0 | Train Loss: 7.7281 | Train Loss Unweighted: 2.5779 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3026 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1176 | Regression_eta: 0.0082 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7908 | Classification_binary: 2.3026 | Regression_cos_phi: 0.4850 | Regression_energy: 1.6258 | Regression_eta: 0.4046 | Regression_pt: 1.7623 | Regression_sin_phi: 0.3569 | LR: 1.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0416 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3105 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3846 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6122.0 | uss_MB: 5355.1 | child_rss_MB: 3980.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:27:19,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:58,734] INFO : Step 43600/50000 rank0 | Train Loss: 7.6677 | Train Loss Unweighted: 2.5158 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2538 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1090 | Regression_eta: 0.0098 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7056 | Classification_binary: 2.2538 | Regression_cos_phi: 0.5962 | Regression_energy: 1.5070 | Regression_eta: 0.4807 | Regression_pt: 1.6400 | Regression_sin_phi: 0.4845 | LR: 1.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6691s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1004 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6122.0 | uss_MB: 5356.2 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:27:58,946] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:28:39,484] INFO : Step 43700/50000 rank0 | Train Loss: 8.0854 | Train Loss Unweighted: 2.5878 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3023 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1203 | Regression_eta: 0.0106 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7629 | Classification_binary: 2.3023 | Regression_cos_phi: 0.5257 | Regression_energy: 1.6642 | Regression_eta: 0.5192 | Regression_pt: 1.7982 | Regression_sin_phi: 0.5129 | LR: 1.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1791 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3981.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:28:39,699] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:18,813] INFO : Step 43800/50000 rank0 | Train Loss: 7.1666 | Train Loss Unweighted: 2.4593 | Train Loss Components: Classification: 0.0153 | Classification_binary: 2.2108 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1050 | Regression_eta: 0.0079 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6412 | Classification_binary: 2.2108 | Regression_cos_phi: 0.5084 | Regression_energy: 1.4520 | Regression_eta: 0.3880 | Regression_pt: 1.5986 | Regression_sin_phi: 0.3676 | LR: 1.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7179s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1528 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3983.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:29:19,025] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:59,377] INFO : Step 43900/50000 rank0 | Train Loss: 8.5324 | Train Loss Unweighted: 2.6790 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3767 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1281 | Regression_eta: 0.0096 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7551 | Classification_binary: 2.3767 | Regression_cos_phi: 0.6793 | Regression_energy: 1.7712 | Regression_eta: 0.4729 | Regression_pt: 1.9009 | Regression_sin_phi: 0.5763 | LR: 1.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7681s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3982.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:29:59,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:39,545] INFO : Step 44000/50000 rank0 | Train Loss: 7.8708 | Train Loss Unweighted: 2.6106 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3341 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1174 | Regression_eta: 0.0096 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.3341 | Regression_cos_phi: 0.5145 | Regression_energy: 1.6231 | Regression_eta: 0.4710 | Regression_pt: 1.7528 | Regression_sin_phi: 0.4198 | LR: 1.32e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6680s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1804 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2802 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3978.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:30:39,764] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:39,782] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 16:30:39,954] INFO : Running validation on rank0 +rank0 [2026-10-28 16:30:39,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:30:39,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:30:41,242] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 16:30:41,406] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:30:41,551] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:30:41,625] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:30:41,693] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:30:41,762] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:30:41,833] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:30:41,902] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:30:42,018] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:30:42,149] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:30:42,281] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:30:42,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:30:42,422] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:30:42,490] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:30:42,560] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:30:42,629] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:30:42,731] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:30:42,738] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:30:42,741] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:30:42,744] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:30:42,745] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:30:42,748] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:30:42,751] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:30:42,752] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:30:42,754] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:30:42,755] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:30:42,756] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:30:42,759] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:30:42,761] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:30:42,762] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:30:42,765] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:30:42,768] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:30:50,059] INFO : VALIDATION | Step=44000/50000 | Train Loss=7.8708 | Valid Loss=7.9560 | Train Loss Unweighted=2.6106 | Valid Loss Unweighted=2.6022 | Stale=0 | ETA=43.4m +rank0 [2026-10-28 16:30:50,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:50,132] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:30:50,135] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:30:50,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:30:52,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:30:52,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:30:53,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:30:54,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:30:54,474] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:30:54,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:54,534] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:30:54,536] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:30:54,655] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:30:56,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:30:57,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:30:57,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:30:58,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:30:58,717] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:30:58,721] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:58,774] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:30:58,777] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:30:58,904] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:31:00,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:31:01,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:31:02,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:31:02,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:31:02,895] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:31:02,903] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:31:03,156] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:31:03,608] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:05,035] INFO : Plotted jets +rank0 [2026-10-28 16:31:06,147] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:06,179] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:31:06,403] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:31:06,774] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:08,145] INFO : Plotted jets +rank0 [2026-10-28 16:31:09,837] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:09,869] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 16:31:10,073] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 16:31:10,489] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:11,872] INFO : Plotted jets +rank0 [2026-10-28 16:31:12,983] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:53,049] INFO : Step 44100/50000 rank0 | Train Loss: 8.0210 | Train Loss Unweighted: 2.6396 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3585 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1190 | Regression_eta: 0.0084 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7840 | Classification_binary: 2.3585 | Regression_cos_phi: 0.5640 | Regression_energy: 1.6455 | Regression_eta: 0.4148 | Regression_pt: 1.7769 | Regression_sin_phi: 0.4773 | LR: 1.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6712s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3977.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:31:53,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:32:32,646] INFO : Step 44200/50000 rank0 | Train Loss: 7.2918 | Train Loss Unweighted: 2.4844 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2436 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1005 | Regression_eta: 0.0102 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7455 | Classification_binary: 2.2436 | Regression_cos_phi: 0.5009 | Regression_energy: 1.3900 | Regression_eta: 0.4994 | Regression_pt: 1.4559 | Regression_sin_phi: 0.4565 | LR: 1.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8042s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0070 | max_forward: 0.0775 | max_loss: 0.0036 | max_backward: 0.3124 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3979.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:32:32,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:12,677] INFO : Step 44300/50000 rank0 | Train Loss: 8.4375 | Train Loss Unweighted: 2.6831 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3849 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1266 | Regression_eta: 0.0117 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7685 | Classification_binary: 2.3849 | Regression_cos_phi: 0.5886 | Regression_energy: 1.7514 | Regression_eta: 0.5739 | Regression_pt: 1.8637 | Regression_sin_phi: 0.5066 | LR: 1.29e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0100 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.3042 | max_optimizer_step: 0.1499 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3980.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:33:12,889] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:53,471] INFO : Step 44400/50000 rank0 | Train Loss: 7.8882 | Train Loss Unweighted: 2.6080 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3327 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1186 | Regression_eta: 0.0094 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8053 | Classification_binary: 2.3327 | Regression_cos_phi: 0.4659 | Regression_energy: 1.6399 | Regression_eta: 0.4628 | Regression_pt: 1.6920 | Regression_sin_phi: 0.4895 | LR: 1.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.0162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1087 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3959 | max_optimizer_step: 0.1477 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 6314.7 | uss_MB: 5551.8 | child_rss_MB: 3985.0 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 16:33:53,681] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:34:33,530] INFO : Step 44500/50000 rank0 | Train Loss: 7.0552 | Train Loss Unweighted: 2.5482 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3085 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1023 | Regression_eta: 0.0069 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7387 | Classification_binary: 2.3085 | Regression_cos_phi: 0.4171 | Regression_energy: 1.4147 | Regression_eta: 0.3406 | Regression_pt: 1.5208 | Regression_sin_phi: 0.3148 | LR: 1.27e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6676s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0325 | max_backward: 0.2035 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3104 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6199.2 | uss_MB: 5433.2 | child_rss_MB: 3979.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:34:33,747] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:13,755] INFO : Step 44600/50000 rank0 | Train Loss: 7.6610 | Train Loss Unweighted: 2.5507 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.2764 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1171 | Regression_eta: 0.0067 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7502 | Classification_binary: 2.2764 | Regression_cos_phi: 0.4720 | Regression_energy: 1.6193 | Regression_eta: 0.3279 | Regression_pt: 1.7711 | Regression_sin_phi: 0.4442 | LR: 1.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1759 | max_loss: 0.0037 | max_backward: 0.2039 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6199.2 | uss_MB: 5434.7 | child_rss_MB: 3979.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:35:13,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:54,380] INFO : Step 44700/50000 rank0 | Train Loss: 7.9455 | Train Loss Unweighted: 2.5134 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2341 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1178 | Regression_eta: 0.0101 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7354 | Classification_binary: 2.2341 | Regression_cos_phi: 0.6096 | Regression_energy: 1.6289 | Regression_eta: 0.4983 | Regression_pt: 1.7462 | Regression_sin_phi: 0.4930 | LR: 1.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1375 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3977.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:35:54,593] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:36:34,361] INFO : Step 44800/50000 rank0 | Train Loss: 7.5848 | Train Loss Unweighted: 2.5759 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3186 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1088 | Regression_eta: 0.0094 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7324 | Classification_binary: 2.3186 | Regression_cos_phi: 0.5660 | Regression_energy: 1.5043 | Regression_eta: 0.4608 | Regression_pt: 1.5950 | Regression_sin_phi: 0.4076 | LR: 1.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7565s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1844 | max_loss: 0.0036 | max_backward: 0.2034 | max_optimizer_step: 0.2368 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3976.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:36:34,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:14,499] INFO : Step 44900/50000 rank0 | Train Loss: 7.9453 | Train Loss Unweighted: 2.6254 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3453 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1201 | Regression_eta: 0.0107 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7118 | Classification_binary: 2.3453 | Regression_cos_phi: 0.4835 | Regression_energy: 1.6612 | Regression_eta: 0.5281 | Regression_pt: 1.7603 | Regression_sin_phi: 0.4550 | LR: 1.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.2705 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:37:14,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:54,203] INFO : Step 45000/50000 rank0 | Train Loss: 7.6674 | Train Loss Unweighted: 2.6009 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3346 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1121 | Regression_eta: 0.0070 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7013 | Classification_binary: 2.3346 | Regression_cos_phi: 0.5637 | Regression_energy: 1.5496 | Regression_eta: 0.3445 | Regression_pt: 1.7157 | Regression_sin_phi: 0.4581 | LR: 1.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7719s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3116 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6200.2 | uss_MB: 5435.0 | child_rss_MB: 3975.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:37:54,418] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:54,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 16:37:54,675] INFO : Running validation on rank0 +rank0 [2026-10-28 16:37:54,675] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:37:54,675] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:37:55,952] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 16:37:56,098] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:37:56,229] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:37:56,302] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:37:56,370] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:37:56,438] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:37:56,506] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:37:56,575] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:37:56,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:37:56,821] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:37:56,952] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:37:57,023] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:37:57,091] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:37:57,158] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:37:57,228] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:37:57,297] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:37:57,398] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:37:57,404] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:37:57,412] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:37:57,420] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:37:57,422] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:37:57,425] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:37:57,426] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:37:57,429] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:37:57,429] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:37:57,432] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:37:57,435] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:37:57,436] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:37:57,439] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:37:57,440] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:37:57,442] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:37:57,443] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:38:04,733] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.6674 | Valid Loss=7.9613 | Train Loss Unweighted=2.6009 | Valid Loss Unweighted=2.6039 | Stale=1 | ETA=36.2m +rank0 [2026-10-28 16:38:04,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:04,805] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:38:04,808] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:04,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:06,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:07,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:08,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:08,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:09,208] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:38:09,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:09,268] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:38:09,271] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:09,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:11,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:11,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:12,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:13,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:13,386] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:38:13,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:13,444] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:38:13,447] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:38:13,560] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:15,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:15,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:16,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:17,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:17,501] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:38:17,509] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:17,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:18,222] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:19,647] INFO : Plotted jets +rank0 [2026-10-28 16:38:20,751] INFO : Plotted jet ratio +rank0 [2026-10-28 16:38:20,783] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:21,010] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:21,387] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:22,755] INFO : Plotted jets +rank0 [2026-10-28 16:38:24,479] INFO : Plotted jet ratio +rank0 [2026-10-28 16:38:24,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 16:38:24,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 16:38:25,133] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:26,516] INFO : Plotted jets +rank0 [2026-10-28 16:38:27,712] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:08,198] INFO : Step 45100/50000 rank0 | Train Loss: 7.7868 | Train Loss Unweighted: 2.6053 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3443 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1076 | Regression_eta: 0.0100 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8143 | Classification_binary: 2.3443 | Regression_cos_phi: 0.5720 | Regression_energy: 1.4877 | Regression_eta: 0.4924 | Regression_pt: 1.6149 | Regression_sin_phi: 0.4612 | LR: 1.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6604s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0864 | max_loss: 0.0036 | max_backward: 0.2041 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:39:08,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:48,199] INFO : Step 45200/50000 rank0 | Train Loss: 7.9533 | Train Loss Unweighted: 2.5811 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3033 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1157 | Regression_eta: 0.0091 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7806 | Classification_binary: 2.3033 | Regression_cos_phi: 0.5579 | Regression_energy: 1.5996 | Regression_eta: 0.4486 | Regression_pt: 1.7628 | Regression_sin_phi: 0.5005 | LR: 1.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1690 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3977.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:39:48,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:40:28,925] INFO : Step 45300/50000 rank0 | Train Loss: 7.6908 | Train Loss Unweighted: 2.5671 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.2890 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1208 | Regression_eta: 0.0080 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7186 | Classification_binary: 2.2890 | Regression_cos_phi: 0.4461 | Regression_energy: 1.6702 | Regression_eta: 0.3910 | Regression_pt: 1.7847 | Regression_sin_phi: 0.3913 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7136s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.2888 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:40:29,140] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:08,535] INFO : Step 45400/50000 rank0 | Train Loss: 7.3959 | Train Loss Unweighted: 2.4640 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.2099 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1071 | Regression_eta: 0.0103 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6630 | Classification_binary: 2.2099 | Regression_cos_phi: 0.5201 | Regression_energy: 1.4816 | Regression_eta: 0.5054 | Regression_pt: 1.5876 | Regression_sin_phi: 0.4283 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8851s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0839 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.3111 | max_optimizer_step: 0.1483 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:41:08,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:48,497] INFO : Step 45500/50000 rank0 | Train Loss: 7.8399 | Train Loss Unweighted: 2.5047 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.2371 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1117 | Regression_eta: 0.0103 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7399 | Classification_binary: 2.2371 | Regression_cos_phi: 0.5699 | Regression_energy: 1.5441 | Regression_eta: 0.5059 | Regression_pt: 1.6379 | Regression_sin_phi: 0.6052 | LR: 1.18e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.8581s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0247 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1494 | max_tensorboard_logging: 0.4030 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6274.3 | uss_MB: 5507.1 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:41:48,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:28,202] INFO : Step 45600/50000 rank0 | Train Loss: 7.7827 | Train Loss Unweighted: 2.5392 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.2732 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1117 | Regression_eta: 0.0116 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7965 | Classification_binary: 2.2732 | Regression_cos_phi: 0.5325 | Regression_energy: 1.5444 | Regression_eta: 0.5706 | Regression_pt: 1.6247 | Regression_sin_phi: 0.4409 | LR: 1.17e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7732s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1796 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 6274.3 | uss_MB: 5508.3 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:42:28,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:08,245] INFO : Step 45700/50000 rank0 | Train Loss: 8.4492 | Train Loss Unweighted: 2.6834 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3851 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1277 | Regression_eta: 0.0131 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7577 | Classification_binary: 2.3851 | Regression_cos_phi: 0.5704 | Regression_energy: 1.7655 | Regression_eta: 0.6436 | Regression_pt: 1.8413 | Regression_sin_phi: 0.4856 | LR: 1.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7181s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1387 | max_loss: 0.0036 | max_backward: 0.2033 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6274.3 | uss_MB: 5508.6 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:43:08,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:49,609] INFO : Step 45800/50000 rank0 | Train Loss: 7.2952 | Train Loss Unweighted: 2.4509 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.1959 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1088 | Regression_eta: 0.0075 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7350 | Classification_binary: 2.1959 | Regression_cos_phi: 0.4653 | Regression_energy: 1.5045 | Regression_eta: 0.3679 | Regression_pt: 1.6077 | Regression_sin_phi: 0.4190 | LR: 1.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0979 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3094 | max_optimizer_step: 0.1484 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 6274.3 | uss_MB: 5508.7 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:43:49,824] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:29,916] INFO : Step 45900/50000 rank0 | Train Loss: 8.3838 | Train Loss Unweighted: 2.5962 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3052 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1213 | Regression_eta: 0.0130 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.3052 | Regression_cos_phi: 0.6942 | Regression_energy: 1.6771 | Regression_eta: 0.6376 | Regression_pt: 1.7957 | Regression_sin_phi: 0.5429 | LR: 1.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7997s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1833 | max_loss: 0.0036 | max_backward: 0.2036 | max_optimizer_step: 0.2410 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6275.3 | uss_MB: 5508.9 | child_rss_MB: 3977.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:44:30,130] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:10,290] INFO : Step 46000/50000 rank0 | Train Loss: 7.9312 | Train Loss Unweighted: 2.6196 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3430 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1171 | Regression_eta: 0.0097 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7335 | Classification_binary: 2.3430 | Regression_cos_phi: 0.5267 | Regression_energy: 1.6196 | Regression_eta: 0.4755 | Regression_pt: 1.7430 | Regression_sin_phi: 0.4899 | LR: 1.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7369s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2569 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.3794 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6275.3 | uss_MB: 5508.8 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:45:10,506] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:10,542] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 16:45:10,722] INFO : Running validation on rank0 +rank0 [2026-10-28 16:45:10,722] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:45:10,722] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:45:12,080] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 16:45:12,224] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:45:12,355] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:45:12,429] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:45:12,497] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:45:12,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:45:12,634] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:45:12,704] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:45:12,820] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:45:12,951] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:45:13,082] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:45:13,153] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:45:13,222] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:45:13,290] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:45:13,359] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:45:13,430] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:45:13,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:45:13,537] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:45:13,539] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:45:13,540] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:45:13,543] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:45:13,544] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:45:13,547] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:45:13,549] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:45:13,550] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:45:13,553] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:45:13,556] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:45:13,557] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:45:13,559] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:45:13,562] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:45:13,570] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:45:13,579] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:45:21,043] INFO : VALIDATION | Step=46000/50000 | Train Loss=7.9312 | Valid Loss=7.9367 | Train Loss Unweighted=2.6196 | Valid Loss Unweighted=2.5952 | Stale=0 | ETA=28.9m +rank0 [2026-10-28 16:45:21,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:21,114] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:45:21,118] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:21,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:23,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:23,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:24,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:25,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:25,414] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:45:25,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:25,473] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:45:25,476] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:25,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:27,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:28,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:28,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:29,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:29,542] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:45:29,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:29,600] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:45:29,603] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:45:29,721] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:31,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:32,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:32,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:33,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:33,721] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:45:33,728] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:33,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:34,431] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:35,854] INFO : Plotted jets +rank0 [2026-10-28 16:45:36,972] INFO : Plotted jet ratio +rank0 [2026-10-28 16:45:37,005] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:37,229] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:37,601] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:39,549] INFO : Plotted jets +rank0 [2026-10-28 16:45:40,623] INFO : Plotted jet ratio +rank0 [2026-10-28 16:45:40,656] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 16:45:40,862] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 16:45:41,277] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:42,658] INFO : Plotted jets +rank0 [2026-10-28 16:45:43,762] INFO : Plotted jet ratio +rank0 [2026-10-28 16:46:24,822] INFO : Step 46100/50000 rank0 | Train Loss: 7.9594 | Train Loss Unweighted: 2.5880 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3016 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1220 | Regression_eta: 0.0087 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7178 | Classification_binary: 2.3016 | Regression_cos_phi: 0.5185 | Regression_energy: 1.6874 | Regression_eta: 0.4255 | Regression_pt: 1.8449 | Regression_sin_phi: 0.4637 | LR: 1.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6421s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0307 | max_backward: 0.2032 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.2947 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3979.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:46:25,037] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:47:04,762] INFO : Step 46200/50000 rank0 | Train Loss: 8.2248 | Train Loss Unweighted: 2.5759 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.2739 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1317 | Regression_eta: 0.0087 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7693 | Classification_binary: 2.2739 | Regression_cos_phi: 0.5840 | Regression_energy: 1.8211 | Regression_eta: 0.4271 | Regression_pt: 1.9068 | Regression_sin_phi: 0.4426 | LR: 1.13e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2567 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3900 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3981.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:47:04,977] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:47:45,355] INFO : Step 46300/50000 rank0 | Train Loss: 7.9768 | Train Loss Unweighted: 2.4779 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.1829 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1262 | Regression_eta: 0.0081 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7637 | Classification_binary: 2.1829 | Regression_cos_phi: 0.5947 | Regression_energy: 1.7458 | Regression_eta: 0.3989 | Regression_pt: 1.9077 | Regression_sin_phi: 0.3830 | LR: 1.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0246 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3111 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3942 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:47:45,569] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:48:26,227] INFO : Step 46400/50000 rank0 | Train Loss: 7.2676 | Train Loss Unweighted: 2.4940 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2479 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1026 | Regression_eta: 0.0085 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7299 | Classification_binary: 2.2479 | Regression_cos_phi: 0.4916 | Regression_energy: 1.4193 | Regression_eta: 0.4173 | Regression_pt: 1.5500 | Regression_sin_phi: 0.4116 | LR: 1.11e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.8336s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1049 | max_loss: 0.0036 | max_backward: 0.3224 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.3393 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6472.1 | uss_MB: 5709.9 | child_rss_MB: 3979.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:48:26,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:06,247] INFO : Step 46500/50000 rank0 | Train Loss: 7.5536 | Train Loss Unweighted: 2.5266 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.2638 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1124 | Regression_eta: 0.0076 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 2.2638 | Regression_cos_phi: 0.5309 | Regression_energy: 1.5546 | Regression_eta: 0.3717 | Regression_pt: 1.6398 | Regression_sin_phi: 0.4033 | LR: 1.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8548s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.4059 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6344.0 | uss_MB: 5579.1 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:49:06,462] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:46,304] INFO : Step 46600/50000 rank0 | Train Loss: 7.9133 | Train Loss Unweighted: 2.5919 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3106 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1198 | Regression_eta: 0.0086 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7153 | Classification_binary: 2.3106 | Regression_cos_phi: 0.5287 | Regression_energy: 1.6570 | Regression_eta: 0.4207 | Regression_pt: 1.7953 | Regression_sin_phi: 0.4856 | LR: 1.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0095 | max_backward: 0.2033 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.2995 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6346.0 | uss_MB: 5580.0 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:49:46,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:26,440] INFO : Step 46700/50000 rank0 | Train Loss: 8.1561 | Train Loss Unweighted: 2.5758 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2807 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1244 | Regression_eta: 0.0098 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7734 | Classification_binary: 2.2807 | Regression_cos_phi: 0.5378 | Regression_energy: 1.7200 | Regression_eta: 0.4815 | Regression_pt: 1.8985 | Regression_sin_phi: 0.4642 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7497s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2378 | max_optimizer_step: 0.1515 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6346.0 | uss_MB: 5580.3 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:26,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:06,936] INFO : Step 46800/50000 rank0 | Train Loss: 7.7938 | Train Loss Unweighted: 2.4785 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.1889 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1274 | Regression_eta: 0.0080 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7351 | Classification_binary: 2.1889 | Regression_cos_phi: 0.5098 | Regression_energy: 1.7623 | Regression_eta: 0.3948 | Regression_pt: 1.8453 | Regression_sin_phi: 0.3575 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.0185s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.2084 | max_loss: 0.0036 | max_backward: 0.3967 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3993 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6346.0 | uss_MB: 5583.0 | child_rss_MB: 3982.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:51:07,150] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:47,507] INFO : Step 46900/50000 rank0 | Train Loss: 7.7673 | Train Loss Unweighted: 2.6570 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3826 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1188 | Regression_eta: 0.0075 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7485 | Classification_binary: 2.3826 | Regression_cos_phi: 0.4940 | Regression_energy: 1.6427 | Regression_eta: 0.3694 | Regression_pt: 1.7478 | Regression_sin_phi: 0.3823 | LR: 1.09e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7716s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2662 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6346.0 | uss_MB: 5581.0 | child_rss_MB: 3980.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:51:47,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:28,504] INFO : Step 47000/50000 rank0 | Train Loss: 7.5421 | Train Loss Unweighted: 2.5632 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.2940 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1158 | Regression_eta: 0.0085 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7788 | Classification_binary: 2.2940 | Regression_cos_phi: 0.3614 | Regression_energy: 1.6018 | Regression_eta: 0.4153 | Regression_pt: 1.7214 | Regression_sin_phi: 0.3693 | LR: 1.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7032s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1003 | max_loss: 0.0036 | max_backward: 0.2042 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6346.0 | uss_MB: 5581.0 | child_rss_MB: 3978.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:28,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:28,736] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 16:52:28,910] INFO : Running validation on rank0 +rank0 [2026-10-28 16:52:28,910] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:52:28,910] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:52:30,197] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 16:52:30,342] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:52:30,472] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:52:30,546] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:52:30,615] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:52:30,685] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:52:30,754] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:52:30,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:52:30,937] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:52:31,068] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:52:31,199] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:52:31,271] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:52:31,339] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:52:31,409] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:52:31,478] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:52:31,547] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:52:31,649] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:52:31,655] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:52:31,657] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:52:31,660] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:52:31,661] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:52:31,664] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:52:31,667] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:52:31,667] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:52:31,670] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:52:31,671] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:52:31,673] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:52:31,674] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:52:31,677] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:52:31,679] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:52:31,680] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:52:31,683] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:52:39,056] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.5421 | Valid Loss=7.9361 | Train Loss Unweighted=2.5632 | Valid Loss Unweighted=2.5935 | Stale=0 | ETA=21.7m +rank0 [2026-10-28 16:52:39,071] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:39,150] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:52:39,154] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:39,358] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:41,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:42,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:42,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:43,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:43,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:43,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:43,836] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:52:43,839] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:43,966] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:45,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:46,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:47,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:47,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:47,997] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:48,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:48,055] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:52:48,058] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:52:48,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:50,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:50,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:51,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:52,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:52,225] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:52,232] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:52,485] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:52,938] INFO : Plotted number of elements +rank0 [2026-10-28 16:52:54,363] INFO : Plotted jets +rank0 [2026-10-28 16:52:55,422] INFO : Plotted jet ratio +rank0 [2026-10-28 16:52:55,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:55,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:56,057] INFO : Plotted number of elements +rank0 [2026-10-28 16:52:57,429] INFO : Plotted jets +rank0 [2026-10-28 16:52:59,120] INFO : Plotted jet ratio +rank0 [2026-10-28 16:52:59,153] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 16:52:59,357] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 16:52:59,770] INFO : Plotted number of elements +rank0 [2026-10-28 16:53:01,152] INFO : Plotted jets +rank0 [2026-10-28 16:53:02,254] INFO : Plotted jet ratio +rank0 [2026-10-28 16:53:42,020] INFO : Step 47100/50000 rank0 | Train Loss: 7.6987 | Train Loss Unweighted: 2.4662 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.1796 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0078 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7068 | Classification_binary: 2.1796 | Regression_cos_phi: 0.4648 | Regression_energy: 1.7289 | Regression_eta: 0.3845 | Regression_pt: 1.8557 | Regression_sin_phi: 0.3786 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0828 | max_backward: 0.2038 | max_optimizer_step: 0.1513 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:53:42,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:22,575] INFO : Step 47200/50000 rank0 | Train Loss: 7.8956 | Train Loss Unweighted: 2.5648 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2881 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1170 | Regression_eta: 0.0089 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.2881 | Regression_cos_phi: 0.5667 | Regression_energy: 1.6178 | Regression_eta: 0.4392 | Regression_pt: 1.7403 | Regression_sin_phi: 0.4879 | LR: 1.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8531s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0289 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3118 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3826 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:54:22,789] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:55:02,771] INFO : Step 47300/50000 rank0 | Train Loss: 7.6050 | Train Loss Unweighted: 2.5608 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3006 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1091 | Regression_eta: 0.0083 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7690 | Classification_binary: 2.3006 | Regression_cos_phi: 0.4950 | Regression_energy: 1.5093 | Regression_eta: 0.4067 | Regression_pt: 1.6296 | Regression_sin_phi: 0.4949 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2289 | max_optimizer_step: 0.1497 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:55:02,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:55:43,505] INFO : Step 47400/50000 rank0 | Train Loss: 8.0405 | Train Loss Unweighted: 2.5912 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3006 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1243 | Regression_eta: 0.0088 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7718 | Classification_binary: 2.3006 | Regression_cos_phi: 0.4731 | Regression_energy: 1.7186 | Regression_eta: 0.4325 | Regression_pt: 1.8584 | Regression_sin_phi: 0.4855 | LR: 1.06e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.2038 | max_optimizer_step: 0.1534 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3977.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:55:43,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:24,077] INFO : Step 47500/50000 rank0 | Train Loss: 8.2188 | Train Loss Unweighted: 2.7061 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4081 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1291 | Regression_eta: 0.0094 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7432 | Classification_binary: 2.4081 | Regression_cos_phi: 0.4736 | Regression_energy: 1.7859 | Regression_eta: 0.4607 | Regression_pt: 1.9103 | Regression_sin_phi: 0.4371 | LR: 1.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8547s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0149 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1524 | max_tensorboard_logging: 0.3992 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6419.3 | uss_MB: 5656.0 | child_rss_MB: 3981.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:24,293] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:04,280] INFO : Step 47600/50000 rank0 | Train Loss: 6.9840 | Train Loss Unweighted: 2.3953 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.1559 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1012 | Regression_eta: 0.0074 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7014 | Classification_binary: 2.1559 | Regression_cos_phi: 0.4680 | Regression_energy: 1.3995 | Regression_eta: 0.3641 | Regression_pt: 1.5078 | Regression_sin_phi: 0.3873 | LR: 1.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8293s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3079 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6420.3 | uss_MB: 5657.0 | child_rss_MB: 3974.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:57:04,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:43,936] INFO : Step 47700/50000 rank0 | Train Loss: 8.5214 | Train Loss Unweighted: 2.6744 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3670 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1320 | Regression_eta: 0.0100 | Regression_pt: 0.1290 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7396 | Classification_binary: 2.3670 | Regression_cos_phi: 0.6070 | Regression_energy: 1.8249 | Regression_eta: 0.4900 | Regression_pt: 1.9412 | Regression_sin_phi: 0.5518 | LR: 1.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7508s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1424 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6420.3 | uss_MB: 5657.3 | child_rss_MB: 3975.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:57:44,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:58:23,662] INFO : Step 47800/50000 rank0 | Train Loss: 7.9297 | Train Loss Unweighted: 2.6392 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3603 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1200 | Regression_eta: 0.0088 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7862 | Classification_binary: 2.3603 | Regression_cos_phi: 0.4835 | Regression_energy: 1.6596 | Regression_eta: 0.4327 | Regression_pt: 1.7449 | Regression_sin_phi: 0.4626 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2126 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.3670 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6421.3 | uss_MB: 5657.3 | child_rss_MB: 3980.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:58:23,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:03,601] INFO : Step 47900/50000 rank0 | Train Loss: 8.0374 | Train Loss Unweighted: 2.5968 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3178 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1161 | Regression_eta: 0.0100 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8021 | Classification_binary: 2.3178 | Regression_cos_phi: 0.5358 | Regression_energy: 1.6060 | Regression_eta: 0.4905 | Regression_pt: 1.7497 | Regression_sin_phi: 0.5355 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7273s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1535 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6421.3 | uss_MB: 5657.3 | child_rss_MB: 3979.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:59:03,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:43,299] INFO : Step 48000/50000 rank0 | Train Loss: 7.5096 | Train Loss Unweighted: 2.5823 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3281 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1073 | Regression_eta: 0.0081 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7800 | Classification_binary: 2.3281 | Regression_cos_phi: 0.5154 | Regression_energy: 1.4843 | Regression_eta: 0.3974 | Regression_pt: 1.5798 | Regression_sin_phi: 0.4247 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.0061s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0938 | max_forward: 0.1053 | max_loss: 0.0038 | max_backward: 0.3941 | max_optimizer_step: 0.1508 | max_tensorboard_logging: 0.3753 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6422.3 | uss_MB: 5660.2 | child_rss_MB: 3977.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:59:43,514] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:43,590] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 16:59:43,765] INFO : Running validation on rank0 +rank0 [2026-10-28 16:59:43,766] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:59:43,766] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:59:45,077] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 16:59:45,221] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:59:45,351] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:59:45,424] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:59:45,492] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:59:45,561] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:59:45,629] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:59:45,698] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:59:45,814] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:59:45,945] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:59:46,077] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:59:46,148] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:59:46,218] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:59:46,286] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:59:46,357] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:59:46,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:59:46,531] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:59:46,538] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:59:46,540] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:59:46,543] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:59:46,544] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:59:46,547] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:59:46,548] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:59:46,550] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:59:46,551] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:59:46,554] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:59:46,556] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:59:46,557] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:59:46,560] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:59:46,562] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:59:46,563] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:59:46,566] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:59:54,040] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.5096 | Valid Loss=7.9276 | Train Loss Unweighted=2.5823 | Valid Loss Unweighted=2.5875 | Stale=0 | ETA=14.5m +rank0 [2026-10-28 16:59:54,050] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:59:54,112] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:59:54,115] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:59:54,300] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:59:56,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:59:56,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:59:57,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:59:58,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:59:58,485] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:59:58,488] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:59:58,546] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:59:58,549] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:59:58,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:00:00,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:00:01,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:00:01,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:00:02,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:00:02,604] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:00:02,607] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:00:02,662] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:00:02,665] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:00:02,789] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:00:04,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:00:05,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:00:05,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:00:06,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:00:06,825] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:00:06,833] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:00:07,087] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:00:07,541] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:08,983] INFO : Plotted jets +rank0 [2026-10-28 17:00:10,110] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:10,142] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:00:10,367] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:00:10,745] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:12,134] INFO : Plotted jets +rank0 [2026-10-28 17:00:13,880] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:13,922] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 17:00:14,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 17:00:14,546] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:15,948] INFO : Plotted jets +rank0 [2026-10-28 17:00:17,064] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:57,143] INFO : Step 48100/50000 rank0 | Train Loss: 7.6490 | Train Loss Unweighted: 2.6035 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3436 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1097 | Regression_eta: 0.0088 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6856 | Classification_binary: 2.3436 | Regression_cos_phi: 0.5542 | Regression_energy: 1.5170 | Regression_eta: 0.4340 | Regression_pt: 1.6279 | Regression_sin_phi: 0.4868 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6679s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0038 | max_backward: 0.2034 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:00:57,356] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:37,199] INFO : Step 48200/50000 rank0 | Train Loss: 7.4357 | Train Loss Unweighted: 2.5648 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3011 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1144 | Regression_eta: 0.0069 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7174 | Classification_binary: 2.3011 | Regression_cos_phi: 0.4042 | Regression_energy: 1.5826 | Regression_eta: 0.3413 | Regression_pt: 1.6869 | Regression_sin_phi: 0.4021 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1635 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3975.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:01:37,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:02:16,950] INFO : Step 48300/50000 rank0 | Train Loss: 7.5699 | Train Loss Unweighted: 2.5425 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2805 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1100 | Regression_eta: 0.0089 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.2805 | Regression_cos_phi: 0.4607 | Regression_energy: 1.5205 | Regression_eta: 0.4362 | Regression_pt: 1.6559 | Regression_sin_phi: 0.4390 | LR: 1.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3045 | max_optimizer_step: 0.1502 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0273 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3977.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:02:17,161] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:02:56,854] INFO : Step 48400/50000 rank0 | Train Loss: 7.7636 | Train Loss Unweighted: 2.5522 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.2848 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1112 | Regression_eta: 0.0094 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8338 | Classification_binary: 2.2848 | Regression_cos_phi: 0.5945 | Regression_energy: 1.5371 | Regression_eta: 0.4625 | Regression_pt: 1.6773 | Regression_sin_phi: 0.3737 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8544s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0046 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1545 | max_tensorboard_logging: 0.3882 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3981.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 17:02:57,068] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:03:36,592] INFO : Step 48500/50000 rank0 | Train Loss: 7.5133 | Train Loss Unweighted: 2.5305 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2774 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1060 | Regression_eta: 0.0077 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7144 | Classification_binary: 2.2774 | Regression_cos_phi: 0.5903 | Regression_energy: 1.4656 | Regression_eta: 0.3772 | Regression_pt: 1.5733 | Regression_sin_phi: 0.5152 | LR: 1.02e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8419s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1828 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6620.1 | uss_MB: 5857.2 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:03:36,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:16,631] INFO : Step 48600/50000 rank0 | Train Loss: 8.4022 | Train Loss Unweighted: 2.5995 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3159 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1159 | Regression_eta: 0.0185 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7931 | Classification_binary: 2.3159 | Regression_cos_phi: 0.5717 | Regression_energy: 1.6033 | Regression_eta: 0.9092 | Regression_pt: 1.6955 | Regression_sin_phi: 0.5137 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7759s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1846 | max_loss: 0.0036 | max_backward: 0.2036 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6492.1 | uss_MB: 5729.3 | child_rss_MB: 3979.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:04:16,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:56,487] INFO : Step 48700/50000 rank0 | Train Loss: 7.9583 | Train Loss Unweighted: 2.5666 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.2963 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1117 | Regression_eta: 0.0107 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8177 | Classification_binary: 2.2963 | Regression_cos_phi: 0.6340 | Regression_energy: 1.5441 | Regression_eta: 0.5265 | Regression_pt: 1.6626 | Regression_sin_phi: 0.4772 | LR: 1.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1043 | max_loss: 0.1093 | max_backward: 0.2876 | max_optimizer_step: 0.1484 | max_tensorboard_logging: 0.2852 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6492.1 | uss_MB: 5731.4 | child_rss_MB: 3978.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:04:56,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:36,414] INFO : Step 48800/50000 rank0 | Train Loss: 7.8042 | Train Loss Unweighted: 2.5026 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2147 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1257 | Regression_eta: 0.0064 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7148 | Classification_binary: 2.2147 | Regression_cos_phi: 0.5287 | Regression_energy: 1.7378 | Regression_eta: 0.3162 | Regression_pt: 1.8532 | Regression_sin_phi: 0.4389 | LR: 1.01e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7694s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2812 | max_optimizer_step: 0.1515 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6492.1 | uss_MB: 5729.8 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:05:36,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:16,659] INFO : Step 48900/50000 rank0 | Train Loss: 7.5086 | Train Loss Unweighted: 2.4769 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.2133 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1119 | Regression_eta: 0.0092 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6968 | Classification_binary: 2.2133 | Regression_cos_phi: 0.4922 | Regression_energy: 1.5474 | Regression_eta: 0.4512 | Regression_pt: 1.6671 | Regression_sin_phi: 0.4407 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7554s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2478 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6492.1 | uss_MB: 5730.2 | child_rss_MB: 3975.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 17:06:16,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:56,281] INFO : Step 49000/50000 rank0 | Train Loss: 7.4955 | Train Loss Unweighted: 2.5158 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.2432 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1186 | Regression_eta: 0.0075 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.2432 | Regression_cos_phi: 0.4272 | Regression_energy: 1.6407 | Regression_eta: 0.3672 | Regression_pt: 1.7621 | Regression_sin_phi: 0.3470 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1836 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.2751 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6492.1 | uss_MB: 5730.2 | child_rss_MB: 3977.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:06:56,495] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:56,513] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 17:06:56,686] INFO : Running validation on rank0 +rank0 [2026-10-28 17:06:56,687] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:06:56,687] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:06:57,986] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 17:06:58,133] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:06:58,265] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:06:58,339] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:06:58,407] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:06:58,476] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:06:58,546] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:06:58,614] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:06:58,729] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:06:58,860] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:06:58,991] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:06:59,062] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:06:59,132] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:06:59,200] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:06:59,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:06:59,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:06:59,437] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:06:59,443] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:06:59,446] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:06:59,448] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:06:59,449] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:06:59,450] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:06:59,453] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:06:59,456] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:06:59,456] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:06:59,459] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:06:59,461] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:06:59,462] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:06:59,465] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:06:59,467] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:06:59,468] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:06:59,471] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:07:07,036] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.4955 | Valid Loss=7.9046 | Train Loss Unweighted=2.5158 | Valid Loss Unweighted=2.5833 | Stale=0 | ETA=7.2m +rank0 [2026-10-28 17:07:07,046] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:07,108] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:07:07,111] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:07,297] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:09,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:09,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:10,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:11,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:11,231] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:07:11,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:11,291] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:07:11,294] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:11,420] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:13,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:14,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:14,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:15,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:15,513] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:07:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:15,571] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:07:15,574] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:07:15,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:17,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:18,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:18,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:19,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:19,655] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:07:19,663] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:19,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:20,373] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:21,799] INFO : Plotted jets +rank0 [2026-10-28 17:07:22,910] INFO : Plotted jet ratio +rank0 [2026-10-28 17:07:22,942] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:23,167] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:23,541] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:25,549] INFO : Plotted jets +rank0 [2026-10-28 17:07:26,608] INFO : Plotted jet ratio +rank0 [2026-10-28 17:07:26,641] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 17:07:26,846] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 17:07:27,263] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:28,642] INFO : Plotted jets +rank0 [2026-10-28 17:07:29,743] INFO : Plotted jet ratio +rank0 [2026-10-28 17:08:10,776] INFO : Step 49100/50000 rank0 | Train Loss: 8.2601 | Train Loss Unweighted: 2.6619 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3761 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1203 | Regression_eta: 0.0117 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7552 | Classification_binary: 2.3761 | Regression_cos_phi: 0.5764 | Regression_energy: 1.6637 | Regression_eta: 0.5773 | Regression_pt: 1.7675 | Regression_sin_phi: 0.5438 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0106 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3106 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6680.6 | uss_MB: 5920.6 | child_rss_MB: 3982.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:08:10,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:08:51,508] INFO : Step 49200/50000 rank0 | Train Loss: 8.2600 | Train Loss Unweighted: 2.5716 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2782 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1244 | Regression_eta: 0.0099 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7540 | Classification_binary: 2.2782 | Regression_cos_phi: 0.6665 | Regression_energy: 1.7202 | Regression_eta: 0.4880 | Regression_pt: 1.8361 | Regression_sin_phi: 0.5170 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.9572s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1046 | max_forward: 0.0777 | max_loss: 0.0037 | max_backward: 0.3116 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6680.6 | uss_MB: 5920.6 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:08:51,724] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:31,249] INFO : Step 49300/50000 rank0 | Train Loss: 7.5964 | Train Loss Unweighted: 2.5544 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.2882 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1126 | Regression_eta: 0.0083 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7665 | Classification_binary: 2.2882 | Regression_cos_phi: 0.3899 | Regression_energy: 1.5572 | Regression_eta: 0.4076 | Regression_pt: 1.6919 | Regression_sin_phi: 0.4951 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0785 | max_loss: 0.0037 | max_backward: 0.2033 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:09:31,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:10:11,268] INFO : Step 49400/50000 rank0 | Train Loss: 7.3948 | Train Loss Unweighted: 2.5036 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2500 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1057 | Regression_eta: 0.0081 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.2500 | Regression_cos_phi: 0.4954 | Regression_energy: 1.4616 | Regression_eta: 0.3979 | Regression_pt: 1.6106 | Regression_sin_phi: 0.4426 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6834s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0037 | max_backward: 0.2035 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.2892 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3976.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:10:11,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:10:51,330] INFO : Step 49500/50000 rank0 | Train Loss: 7.5640 | Train Loss Unweighted: 2.4239 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.1548 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1153 | Regression_eta: 0.0100 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7058 | Classification_binary: 2.1548 | Regression_cos_phi: 0.5089 | Regression_energy: 1.5942 | Regression_eta: 0.4921 | Regression_pt: 1.6839 | Regression_sin_phi: 0.4243 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0037 | max_backward: 0.2043 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.2926 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3976.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:10:51,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:31,775] INFO : Step 49600/50000 rank0 | Train Loss: 7.7997 | Train Loss Unweighted: 2.5712 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1256 | Regression_eta: 0.0090 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6921 | Classification_binary: 2.2874 | Regression_cos_phi: 0.4427 | Regression_energy: 1.7368 | Regression_eta: 0.4407 | Regression_pt: 1.7866 | Regression_sin_phi: 0.4134 | LR: 1.00e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.7940s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1054 | max_loss: 0.0039 | max_backward: 0.3198 | max_optimizer_step: 0.1519 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6561.0 | uss_MB: 5803.9 | child_rss_MB: 3980.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:11:31,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:11,844] INFO : Step 49700/50000 rank0 | Train Loss: 7.8176 | Train Loss Unweighted: 2.5364 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2590 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1192 | Regression_eta: 0.0085 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7522 | Classification_binary: 2.2590 | Regression_cos_phi: 0.5899 | Regression_energy: 1.6480 | Regression_eta: 0.4194 | Regression_pt: 1.7417 | Regression_sin_phi: 0.4074 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6790s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0570 | max_backward: 0.2035 | max_optimizer_step: 0.1537 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6561.0 | uss_MB: 5801.9 | child_rss_MB: 3978.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:12:12,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:51,718] INFO : Step 49800/50000 rank0 | Train Loss: 8.1509 | Train Loss Unweighted: 2.5941 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3054 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1208 | Regression_eta: 0.0105 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8510 | Classification_binary: 2.3054 | Regression_cos_phi: 0.5099 | Regression_energy: 1.6710 | Regression_eta: 0.5168 | Regression_pt: 1.8209 | Regression_sin_phi: 0.4760 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8536s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0301 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3099 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6561.0 | uss_MB: 5802.2 | child_rss_MB: 3976.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:12:51,932] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:13:32,802] INFO : Step 49900/50000 rank0 | Train Loss: 8.0252 | Train Loss Unweighted: 2.6481 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3611 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1236 | Regression_eta: 0.0092 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7154 | Classification_binary: 2.3611 | Regression_cos_phi: 0.5045 | Regression_energy: 1.7097 | Regression_eta: 0.4535 | Regression_pt: 1.8282 | Regression_sin_phi: 0.4528 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8208s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0038 | max_forward: 0.0784 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6561.0 | uss_MB: 5802.2 | child_rss_MB: 3978.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:13:33,016] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:13,250] INFO : Step 50000/50000 rank0 | Train Loss: 7.4666 | Train Loss Unweighted: 2.4566 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.1871 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1153 | Regression_eta: 0.0071 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6603 | Classification_binary: 2.1871 | Regression_cos_phi: 0.5033 | Regression_energy: 1.5945 | Regression_eta: 0.3488 | Regression_pt: 1.7513 | Regression_sin_phi: 0.4214 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1839 | max_loss: 0.0035 | max_backward: 0.2039 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6561.0 | uss_MB: 5802.3 | child_rss_MB: 3980.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:14:13,462] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:13,480] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 17:14:13,648] INFO : Running validation on rank0 +rank0 [2026-10-28 17:14:13,649] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:14:13,649] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:14:14,956] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 17:14:15,099] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:14:15,231] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:14:15,305] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:14:15,373] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:14:15,442] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:14:15,511] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:14:15,580] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:14:15,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:14:15,826] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:14:15,958] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:14:16,030] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:14:16,100] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:14:16,168] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:14:16,237] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:14:16,306] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:14:16,406] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:14:16,412] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:14:16,415] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:14:16,416] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:14:16,419] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:14:16,420] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:14:16,422] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:14:16,425] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:14:16,426] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:14:16,429] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:14:16,431] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:14:16,432] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:14:16,435] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:14:16,437] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:14:16,440] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:14:16,441] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:14:24,047] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.4666 | Valid Loss=7.8930 | Train Loss Unweighted=2.4566 | Valid Loss Unweighted=2.5849 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 17:14:24,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:24,118] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:14:24,121] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:24,304] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:26,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:27,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:27,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:28,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:28,576] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:28,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:28,636] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:14:28,639] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:28,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:30,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:31,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:32,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:32,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:32,849] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:32,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:32,906] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:14:32,909] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:14:33,029] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:35,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:35,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:36,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:36,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:37,070] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:37,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:37,329] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:37,780] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:39,202] INFO : Plotted jets +rank0 [2026-10-28 17:14:40,320] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:40,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:40,572] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:40,947] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:42,319] INFO : Plotted jets +rank0 [2026-10-28 17:14:44,000] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:44,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 17:14:44,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 17:14:44,645] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:46,031] INFO : Plotted jets +rank0 [2026-10-28 17:14:47,128] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:47,737] INFO : Training completed. Total time on device 0: 362.238min diff --git a/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log.0 b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..2a3a43403b52d723cdd0379a1fdf714fadcf1b31 --- /dev/null +++ b/clic/clusters/v3.2.0/clic_pf_heptv2_seed12345_20261028_111202_849554/train.log.0 @@ -0,0 +1,13929 @@ +rank0 [2026-10-28 11:12:24,884] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-10-28 11:12:24,885] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-10-28 11:12:24,885] INFO : Instantiating model +rank0 [2026-10-28 11:12:24,885] INFO : MLPF __init__ conv_type=ModelType.HEPTV2 num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=19 +rank0 [2026-10-28 11:12:24,886] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-10-28 11:12:24,922] INFO : Input encoding initialization took 0.04s +rank0 [2026-10-28 11:12:24,923] INFO : nn0 parameters: 137216 +rank0 [2026-10-28 11:12:24,923] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-10-28 11:12:24,923] INFO : Initializing backbone layer 0 +rank0 [2026-10-28 11:12:24,976] INFO : Initializing backbone layer 1 +rank0 [2026-10-28 11:12:24,983] INFO : Initializing backbone layer 2 +rank0 [2026-10-28 11:12:24,991] INFO : Initializing backbone layer 3 +rank0 [2026-10-28 11:12:24,998] INFO : Initializing backbone layer 4 +rank0 [2026-10-28 11:12:25,006] INFO : Initializing backbone layer 5 +rank0 [2026-10-28 11:12:25,013] INFO : Backbone initialization took 0.09s +rank0 [2026-10-28 11:12:25,014] INFO : backbone parameters: 7901376 +rank0 [2026-10-28 11:12:25,014] INFO : Initializing output DNNs +rank0 [2026-10-28 11:12:25,018] INFO : Output DNNs initialization took 0.00s +rank0 [2026-10-28 11:12:25,019] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-10-28 11:12:25,019] INFO : jagged_attention enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : flash_attn_varlen enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : task_queries enabled=False +rank0 [2026-10-28 11:12:25,019] INFO : classification_norm parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_norm parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : classification_query parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_query parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : classification_readout parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : regression_readout parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : output_mode=elementwise +rank0 [2026-10-28 11:12:25,019] INFO : set_decoder parameters: 0 +rank0 [2026-10-28 11:12:25,019] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_pid parameters: 67846 +rank0 [2026-10-28 11:12:25,020] INFO : nn_pt parameters: 133122 +rank0 [2026-10-28 11:12:25,020] INFO : nn_eta parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-10-28 11:12:25,020] INFO : nn_energy parameters: 133122 +rank0 [2026-10-28 11:12:25,021] INFO : Total MLPF parameters: 8639954 +rank0 [2026-10-28 11:12:25,021] INFO : MLPF __init__ done +rank0 [2026-10-28 11:12:25,021] INFO : Instantiated model +rank0 [2026-10-28 11:12:25,021] INFO : Moving model to device rank=0 +rank0 [2026-10-28 11:12:25,241] INFO : Moved model to device rank=0 +rank0 [2026-10-28 11:12:25,242] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.001 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-10-28 11:12:25,356] INFO : Modules Trainable parameters Non-trainable parameters +0 nn0.0.weight 4864 0 +1 nn0.0.bias 256 0 +2 nn0.2.weight 256 0 +3 nn0.2.bias 256 0 +4 nn0.4.weight 131072 0 +.. ... ... ... +205 nn_energy.nn.1.0.bias 256 0 +206 nn_energy.nn.1.2.weight 256 0 +207 nn_energy.nn.1.2.bias 256 0 +208 nn_energy.nn.1.4.weight 256 0 +209 nn_energy.nn.1.4.bias 1 0 + +[210 rows x 3 columns] +rank0 [2026-10-28 11:12:25,356] INFO : MLPF( + (feature_engineering): Identity() + (nn0): Sequential( + (0): Linear(in_features=19, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x HEPTv2Layer( + embedding_dim=256, num_heads=16, width=256, dropout=0.0, block_size=128, n_hashes=3, num_regions=140, num_w_per_dist=10 + (attn): HEPTv2Attention( + (out_linear): Linear(in_features=256, out_features=256, bias=True) + (e2lsh): E2LSH() + (e2lsh_new): E2LSH() + ) + (w_q): Linear(in_features=256, out_features=256, bias=False) + (w_k): Linear(in_features=256, out_features=256, bias=False) + (w_v): Linear(in_features=256, out_features=256, bias=False) + (w_rpe): Linear(in_features=10, out_features=256, bias=True) + (pe_func): PELearned( + (position_embedding_head): Sequential( + (0): Linear(in_features=2, out_features=1024, bias=True) + (1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True) + (2): SiLU() + (3): Linear(in_features=1024, out_features=256, bias=True) + ) + ) + (norm1): Qwen3RMSNorm() + (norm2): Qwen3RMSNorm() + (q_norm): Qwen3RMSNorm() + (k_norm): Qwen3RMSNorm() + (ff): Qwen3MLP( + (gate_proj): Linear(in_features=256, out_features=1024, bias=False) + (up_proj): Linear(in_features=256, out_features=1024, bias=False) + (down_proj): Linear(in_features=1024, out_features=256, bias=False) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-10-28 11:12:25,357] INFO : Trainable parameters: 8639954 +rank0 [2026-10-28 11:12:25,357] INFO : Non-trainable parameters: 23268 +rank0 [2026-10-28 11:12:25,358] INFO : Total parameters: 8663222 +rank0 [2026-10-28 11:12:25,360] INFO : Modules Trainable parameters Non-trainable parameters + nn0.0.weight 4864 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.regions 0 96 + backbone.0.attn.out_linear.weight 65536 0 + backbone.0.attn.out_linear.bias 256 0 + backbone.0.attn.e2lsh.alpha 0 864 + backbone.0.attn.e2lsh.beta 0 3 + backbone.0.attn.e2lsh_new.alpha 0 96 + backbone.0.attn.e2lsh_new.beta 0 3 + backbone.0.w_q.weight 65536 0 + backbone.0.w_k.weight 65536 0 + backbone.0.w_v.weight 65536 0 + backbone.0.w_rpe.weight 0 2560 + backbone.0.w_rpe.bias 0 256 +backbone.0.pe_func.position_embedding_head.0.weight 2048 0 + backbone.0.pe_func.position_embedding_head.0.bias 1024 0 +backbone.0.pe_func.position_embedding_head.1.weight 1024 0 + backbone.0.pe_func.position_embedding_head.1.bias 1024 0 +backbone.0.pe_func.position_embedding_head.3.weight 262144 0 + backbone.0.pe_func.position_embedding_head.3.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm2.weight 256 0 + backbone.0.q_norm.weight 16 0 + backbone.0.k_norm.weight 16 0 + backbone.0.ff.gate_proj.weight 262144 0 + backbone.0.ff.up_proj.weight 262144 0 + backbone.0.ff.down_proj.weight 262144 0 + backbone.1.regions 0 96 + backbone.1.attn.out_linear.weight 65536 0 + backbone.1.attn.out_linear.bias 256 0 + backbone.1.attn.e2lsh.alpha 0 864 + backbone.1.attn.e2lsh.beta 0 3 + backbone.1.attn.e2lsh_new.alpha 0 96 + backbone.1.attn.e2lsh_new.beta 0 3 + backbone.1.w_q.weight 65536 0 + backbone.1.w_k.weight 65536 0 + backbone.1.w_v.weight 65536 0 + backbone.1.w_rpe.weight 0 2560 + backbone.1.w_rpe.bias 0 256 +backbone.1.pe_func.position_embedding_head.0.weight 2048 0 + backbone.1.pe_func.position_embedding_head.0.bias 1024 0 +backbone.1.pe_func.position_embedding_head.1.weight 1024 0 + backbone.1.pe_func.position_embedding_head.1.bias 1024 0 +backbone.1.pe_func.position_embedding_head.3.weight 262144 0 + backbone.1.pe_func.position_embedding_head.3.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm2.weight 256 0 + backbone.1.q_norm.weight 16 0 + backbone.1.k_norm.weight 16 0 + backbone.1.ff.gate_proj.weight 262144 0 + backbone.1.ff.up_proj.weight 262144 0 + backbone.1.ff.down_proj.weight 262144 0 + backbone.2.regions 0 96 + backbone.2.attn.out_linear.weight 65536 0 + backbone.2.attn.out_linear.bias 256 0 + backbone.2.attn.e2lsh.alpha 0 864 + backbone.2.attn.e2lsh.beta 0 3 + backbone.2.attn.e2lsh_new.alpha 0 96 + backbone.2.attn.e2lsh_new.beta 0 3 + backbone.2.w_q.weight 65536 0 + backbone.2.w_k.weight 65536 0 + backbone.2.w_v.weight 65536 0 + backbone.2.w_rpe.weight 0 2560 + backbone.2.w_rpe.bias 0 256 +backbone.2.pe_func.position_embedding_head.0.weight 2048 0 + backbone.2.pe_func.position_embedding_head.0.bias 1024 0 +backbone.2.pe_func.position_embedding_head.1.weight 1024 0 + backbone.2.pe_func.position_embedding_head.1.bias 1024 0 +backbone.2.pe_func.position_embedding_head.3.weight 262144 0 + backbone.2.pe_func.position_embedding_head.3.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm2.weight 256 0 + backbone.2.q_norm.weight 16 0 + backbone.2.k_norm.weight 16 0 + backbone.2.ff.gate_proj.weight 262144 0 + backbone.2.ff.up_proj.weight 262144 0 + backbone.2.ff.down_proj.weight 262144 0 + backbone.3.regions 0 96 + backbone.3.attn.out_linear.weight 65536 0 + backbone.3.attn.out_linear.bias 256 0 + backbone.3.attn.e2lsh.alpha 0 864 + backbone.3.attn.e2lsh.beta 0 3 + backbone.3.attn.e2lsh_new.alpha 0 96 + backbone.3.attn.e2lsh_new.beta 0 3 + backbone.3.w_q.weight 65536 0 + backbone.3.w_k.weight 65536 0 + backbone.3.w_v.weight 65536 0 + backbone.3.w_rpe.weight 0 2560 + backbone.3.w_rpe.bias 0 256 +backbone.3.pe_func.position_embedding_head.0.weight 2048 0 + backbone.3.pe_func.position_embedding_head.0.bias 1024 0 +backbone.3.pe_func.position_embedding_head.1.weight 1024 0 + backbone.3.pe_func.position_embedding_head.1.bias 1024 0 +backbone.3.pe_func.position_embedding_head.3.weight 262144 0 + backbone.3.pe_func.position_embedding_head.3.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm2.weight 256 0 + backbone.3.q_norm.weight 16 0 + backbone.3.k_norm.weight 16 0 + backbone.3.ff.gate_proj.weight 262144 0 + backbone.3.ff.up_proj.weight 262144 0 + backbone.3.ff.down_proj.weight 262144 0 + backbone.4.regions 0 96 + backbone.4.attn.out_linear.weight 65536 0 + backbone.4.attn.out_linear.bias 256 0 + backbone.4.attn.e2lsh.alpha 0 864 + backbone.4.attn.e2lsh.beta 0 3 + backbone.4.attn.e2lsh_new.alpha 0 96 + backbone.4.attn.e2lsh_new.beta 0 3 + backbone.4.w_q.weight 65536 0 + backbone.4.w_k.weight 65536 0 + backbone.4.w_v.weight 65536 0 + backbone.4.w_rpe.weight 0 2560 + backbone.4.w_rpe.bias 0 256 +backbone.4.pe_func.position_embedding_head.0.weight 2048 0 + backbone.4.pe_func.position_embedding_head.0.bias 1024 0 +backbone.4.pe_func.position_embedding_head.1.weight 1024 0 + backbone.4.pe_func.position_embedding_head.1.bias 1024 0 +backbone.4.pe_func.position_embedding_head.3.weight 262144 0 + backbone.4.pe_func.position_embedding_head.3.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm2.weight 256 0 + backbone.4.q_norm.weight 16 0 + backbone.4.k_norm.weight 16 0 + backbone.4.ff.gate_proj.weight 262144 0 + backbone.4.ff.up_proj.weight 262144 0 + backbone.4.ff.down_proj.weight 262144 0 + backbone.5.regions 0 96 + backbone.5.attn.out_linear.weight 65536 0 + backbone.5.attn.out_linear.bias 256 0 + backbone.5.attn.e2lsh.alpha 0 864 + backbone.5.attn.e2lsh.beta 0 3 + backbone.5.attn.e2lsh_new.alpha 0 96 + backbone.5.attn.e2lsh_new.beta 0 3 + backbone.5.w_q.weight 65536 0 + backbone.5.w_k.weight 65536 0 + backbone.5.w_v.weight 65536 0 + backbone.5.w_rpe.weight 0 2560 + backbone.5.w_rpe.bias 0 256 +backbone.5.pe_func.position_embedding_head.0.weight 2048 0 + backbone.5.pe_func.position_embedding_head.0.bias 1024 0 +backbone.5.pe_func.position_embedding_head.1.weight 1024 0 + backbone.5.pe_func.position_embedding_head.1.bias 1024 0 +backbone.5.pe_func.position_embedding_head.3.weight 262144 0 + backbone.5.pe_func.position_embedding_head.3.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm2.weight 256 0 + backbone.5.q_norm.weight 16 0 + backbone.5.k_norm.weight 16 0 + backbone.5.ff.gate_proj.weight 262144 0 + backbone.5.ff.up_proj.weight 262144 0 + backbone.5.ff.down_proj.weight 262144 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-10-28 11:12:25,364] INFO : Creating experiment dir /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554 +rank0 [2026-10-28 11:12:25,364] INFO : Model directory /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554 +rank0 [2026-10-28 11:12:25,364] INFO : Getting dataloaders +rank0 [2026-10-28 11:12:25,364] INFO : sample=clic_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,271] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,313] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,354] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,395] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,417] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,473] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,512] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,549] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,590] INFO : train_dataset: clic_edm_ttbar_pf, 90000 +rank0 [2026-10-28 11:12:32,614] INFO : train_dataset: clic_edm_ttbar_pf, 90900 +rank0 [2026-10-28 11:12:32,614] INFO : sample=clic_edm_ww_fullhad_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,650] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,679] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,703] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,745] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,783] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,814] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,842] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,877] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,900] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90000 +rank0 [2026-10-28 11:12:32,935] INFO : train_dataset: clic_edm_ww_fullhad_pf, 90900 +rank0 [2026-10-28 11:12:32,935] INFO : sample=clic_edm_qq_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:32,973] INFO : train_dataset: clic_edm_qq_pf, 90000 +rank0 [2026-10-28 11:12:33,011] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,082] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,102] INFO : train_dataset: clic_edm_qq_pf, 89999 +rank0 [2026-10-28 11:12:33,123] INFO : train_dataset: clic_edm_qq_pf, 89997 +rank0 [2026-10-28 11:12:33,172] INFO : train_dataset: clic_edm_qq_pf, 89996 +rank0 [2026-10-28 11:12:33,198] INFO : train_dataset: clic_edm_qq_pf, 89998 +rank0 [2026-10-28 11:12:33,229] INFO : train_dataset: clic_edm_qq_pf, 89994 +rank0 [2026-10-28 11:12:33,263] INFO : train_dataset: clic_edm_qq_pf, 89995 +rank0 [2026-10-28 11:12:33,278] INFO : train_dataset: clic_edm_qq_pf, 90099 +rank0 [2026-10-28 11:12:33,278] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-10-28 11:12:33,278] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,278] INFO : train mix: physical: weight=21108 step_share=1.000 batches_per_epoch=21108 events_per_step=128 expected_passes_in_50000_steps=2.37 +rank0 [2026-10-28 11:12:33,278] INFO : sample=clic_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,284] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,290] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,296] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,302] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,307] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,313] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,319] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,325] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,331] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,337] INFO : valid_dataset: clic_edm_ttbar_pf, 51 +rank0 [2026-10-28 11:12:33,337] INFO : sample=clic_edm_ww_fullhad_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,342] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,347] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,352] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,357] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,363] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,368] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,373] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,378] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,383] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : valid_dataset: clic_edm_ww_fullhad_pf, 51 +rank0 [2026-10-28 11:12:33,388] INFO : sample=clic_edm_qq_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:12:33,393] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,398] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,403] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,408] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,413] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,418] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,423] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,428] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,434] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,438] INFO : valid_dataset: clic_edm_qq_pf, 51 +rank0 [2026-10-28 11:12:33,439] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-10-28 11:12:33,439] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-10-28 11:12:33,466] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,469] INFO : Got dataloaders +rank0 [2026-10-28 11:12:33,470] INFO : loader split=train rank=0 len=21108 +rank0 [2026-10-28 11:12:33,470] INFO : loader split=valid rank=0 len=12 +rank0 [2026-10-28 11:12:33,470] INFO : Starting training from step 1 to 50000 on rank 0 of 1 +rank0 [2026-10-28 11:12:33,476] INFO : Creating train_iterator +rank0 [2026-10-28 11:12:33,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 11:12:33,494] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:12:33,747] INFO : Created train_iterator +rank0 [2026-10-28 11:13:44,783] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 4.85045928478241, 'Classification': 0.11577398337423801, 'Regression_pt': 0.32241056829690934, 'Regression_eta': 0.09871801823377609, 'Regression_sin_phi': 0.0849964850768447, 'Regression_cos_phi': 0.0726868094317615, 'Regression_energy': 0.3507564795017242}, weights={'Classification_binary': 1.0, 'Classification': 41.89593505859375, 'Regression_pt': 15.044355392456055, 'Regression_eta': 49.13448715209961, 'Regression_sin_phi': 57.066585540771484, 'Regression_cos_phi': 66.73094177246094, 'Regression_energy': 13.828566551208496} +rank0 [2026-10-28 11:13:45,975] INFO : Step 100/50000 rank0 | Train Loss: 5.1883 | Train Loss Unweighted: 5.1883 | Train Loss Components: Classification: 0.0537 | Classification_binary: 4.4025 | Regression_cos_phi: 0.0422 | Regression_energy: 0.3073 | Regression_eta: 0.0481 | Regression_pt: 0.3016 | Regression_sin_phi: 0.0330 | Task Weights: Classification: 1.0000 | Classification_binary: 1.0000 | Regression_cos_phi: 1.0000 | Regression_energy: 1.0000 | Regression_eta: 1.0000 | Regression_pt: 1.0000 | Regression_sin_phi: 1.0000 | Task Weighted Losses: Classification: 0.0537 | Classification_binary: 4.4025 | Regression_cos_phi: 0.0422 | Regression_energy: 0.3073 | Regression_eta: 0.0481 | Regression_pt: 0.3016 | Regression_sin_phi: 0.0330 | LR: 1.00e-03 | DataLoad Time: 0.0021s | Model Forward Time: 1.6084s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0516 | max_backward: 0.2038 | max_optimizer_step: 0.1187 | max_tensorboard_logging: 0.4370 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 2956.1 | uss_MB: 2130.6 | child_rss_MB: 18335.3 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:13:46,193] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:14:25,886] INFO : Step 200/50000 rank0 | Train Loss: 16.9962 | Train Loss Unweighted: 4.8663 | Train Loss Components: Classification: 0.0438 | Classification_binary: 4.2706 | Regression_cos_phi: 0.0230 | Regression_energy: 0.2439 | Regression_eta: 0.0279 | Regression_pt: 0.2395 | Regression_sin_phi: 0.0177 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.8349 | Classification_binary: 4.2706 | Regression_cos_phi: 1.5358 | Regression_energy: 3.3722 | Regression_eta: 1.3692 | Regression_pt: 3.6034 | Regression_sin_phi: 1.0102 | LR: 1.00e-03 | DataLoad Time: 0.0025s | Model Forward Time: 1.3899s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1847 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.1026 | max_tensorboard_logging: 0.2732 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2955.8 | uss_MB: 2135.4 | child_rss_MB: 4121.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:14:26,102] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:05,302] INFO : Step 300/50000 rank0 | Train Loss: 16.2052 | Train Loss Unweighted: 4.6492 | Train Loss Components: Classification: 0.0407 | Classification_binary: 4.0329 | Regression_cos_phi: 0.0158 | Regression_energy: 0.2619 | Regression_eta: 0.0186 | Regression_pt: 0.2631 | Regression_sin_phi: 0.0160 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.7067 | Classification_binary: 4.0329 | Regression_cos_phi: 1.0536 | Regression_energy: 3.6220 | Regression_eta: 0.9162 | Regression_pt: 3.9587 | Regression_sin_phi: 0.9151 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4383s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0772 | max_loss: 0.0029 | max_backward: 0.3085 | max_optimizer_step: 0.1038 | max_tensorboard_logging: 0.3489 | max_tensorboard_flush: 0.0080 | Memory: rss_MB: 2955.8 | uss_MB: 2135.4 | child_rss_MB: 4145.8 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:15:05,516] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:15:44,934] INFO : Step 400/50000 rank0 | Train Loss: 13.6138 | Train Loss Unweighted: 4.5148 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.0488 | Regression_cos_phi: 0.0131 | Regression_energy: 0.1931 | Regression_eta: 0.0148 | Regression_pt: 0.1917 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.7460 | Classification_binary: 4.0488 | Regression_cos_phi: 0.8709 | Regression_energy: 2.6705 | Regression_eta: 0.7272 | Regression_pt: 2.8833 | Regression_sin_phi: 0.6671 | LR: 1.00e-03 | DataLoad Time: 0.0018s | Model Forward Time: 1.5434s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0032 | max_backward: 0.3822 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.3636 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 2957.5 | uss_MB: 2137.5 | child_rss_MB: 4115.6 | allocated_MB: 295.1 | reserved_MB: 3800.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 11:15:45,146] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:16:25,370] INFO : Step 500/50000 rank0 | Train Loss: 14.5518 | Train Loss Unweighted: 4.4666 | Train Loss Components: Classification: 0.0392 | Classification_binary: 3.9419 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2201 | Regression_eta: 0.0197 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6435 | Classification_binary: 3.9419 | Regression_cos_phi: 0.8466 | Regression_energy: 3.0437 | Regression_eta: 0.9661 | Regression_pt: 3.2879 | Regression_sin_phi: 0.8220 | LR: 1.00e-03 | DataLoad Time: 0.0020s | Model Forward Time: 1.4333s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1048 | max_forward: 0.0777 | max_loss: 0.0029 | max_backward: 0.2038 | max_optimizer_step: 0.1948 | max_tensorboard_logging: 0.2531 | max_tensorboard_flush: 0.0084 | Memory: rss_MB: 2955.2 | uss_MB: 2135.4 | child_rss_MB: 4113.6 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:16:25,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:05,330] INFO : Step 600/50000 rank0 | Train Loss: 15.3389 | Train Loss Unweighted: 4.6861 | Train Loss Components: Classification: 0.0405 | Classification_binary: 4.1250 | Regression_cos_phi: 0.0134 | Regression_energy: 0.2384 | Regression_eta: 0.0204 | Regression_pt: 0.2344 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6955 | Classification_binary: 4.1250 | Regression_cos_phi: 0.8952 | Regression_energy: 3.2969 | Regression_eta: 1.0005 | Regression_pt: 3.5257 | Regression_sin_phi: 0.8000 | LR: 1.00e-03 | DataLoad Time: 0.0019s | Model Forward Time: 1.4723s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1066 | max_tensorboard_logging: 0.3644 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 2892.7 | uss_MB: 2073.6 | child_rss_MB: 4109.0 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:17:05,543] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:17:44,967] INFO : Step 700/50000 rank0 | Train Loss: 13.9769 | Train Loss Unweighted: 4.4707 | Train Loss Components: Classification: 0.0397 | Classification_binary: 3.9779 | Regression_cos_phi: 0.0138 | Regression_energy: 0.2076 | Regression_eta: 0.0140 | Regression_pt: 0.2039 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6613 | Classification_binary: 3.9779 | Regression_cos_phi: 0.9199 | Regression_energy: 2.8704 | Regression_eta: 0.6874 | Regression_pt: 3.0670 | Regression_sin_phi: 0.7929 | LR: 1.00e-03 | DataLoad Time: 0.0022s | Model Forward Time: 1.4228s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0703 | max_backward: 0.2035 | max_optimizer_step: 0.1016 | max_tensorboard_logging: 0.3675 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 2892.6 | uss_MB: 2074.6 | child_rss_MB: 4110.5 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:17:45,181] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:18:25,118] INFO : Step 800/50000 rank0 | Train Loss: 13.4356 | Train Loss Unweighted: 4.3883 | Train Loss Components: Classification: 0.0376 | Classification_binary: 3.9055 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2047 | Regression_eta: 0.0138 | Regression_pt: 0.2044 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5736 | Classification_binary: 3.9055 | Regression_cos_phi: 0.7061 | Regression_energy: 2.8310 | Regression_eta: 0.6765 | Regression_pt: 3.0749 | Regression_sin_phi: 0.6678 | LR: 9.99e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4582s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3107 | max_optimizer_step: 0.1036 | max_tensorboard_logging: 0.3634 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 2895.4 | uss_MB: 2076.2 | child_rss_MB: 4113.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:18:25,333] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:05,318] INFO : Step 900/50000 rank0 | Train Loss: 13.7821 | Train Loss Unweighted: 4.3842 | Train Loss Components: Classification: 0.0391 | Classification_binary: 3.8885 | Regression_cos_phi: 0.0107 | Regression_energy: 0.2093 | Regression_eta: 0.0150 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.6371 | Classification_binary: 3.8885 | Regression_cos_phi: 0.7142 | Regression_energy: 2.8946 | Regression_eta: 0.7358 | Regression_pt: 3.1292 | Regression_sin_phi: 0.7827 | LR: 9.99e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.3648s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3086 | max_optimizer_step: 0.1025 | max_tensorboard_logging: 0.2728 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 2895.4 | uss_MB: 2076.2 | child_rss_MB: 4112.4 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:19:05,530] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:44,801] INFO : Step 1000/50000 rank0 | Train Loss: 13.0955 | Train Loss Unweighted: 4.3589 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.8991 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1933 | Regression_eta: 0.0146 | Regression_pt: 0.1936 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4931 | Classification_binary: 3.8991 | Regression_cos_phi: 0.7407 | Regression_energy: 2.6735 | Regression_eta: 0.7169 | Regression_pt: 2.9129 | Regression_sin_phi: 0.6594 | LR: 9.99e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.3523s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0030 | max_backward: 0.3049 | max_optimizer_step: 0.1051 | max_tensorboard_logging: 0.2636 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 2895.5 | uss_MB: 2076.2 | child_rss_MB: 4109.1 | allocated_MB: 291.4 | reserved_MB: 3800.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:19:45,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 6% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:19:45,032] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-1000.pth +rank0 [2026-10-28 11:19:45,208] INFO : Running validation on rank0 +rank0 [2026-10-28 11:19:45,208] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:19:45,208] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:19:47,539] INFO : saved batch0_epoch1000.parquet +rank0 [2026-10-28 11:19:47,723] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:19:47,821] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:19:47,880] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:19:47,940] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:19:47,989] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:19:48,049] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:19:48,110] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:19:48,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:19:48,260] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:19:48,358] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:19:48,419] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:19:48,482] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:19:48,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:19:48,595] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:19:48,655] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:19:48,713] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:19:48,717] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:19:48,720] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:19:48,723] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:19:48,723] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:19:48,726] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:19:48,729] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:19:48,732] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:19:48,735] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:19:48,738] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:19:48,739] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:19:48,741] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:19:48,744] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:19:48,747] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:19:48,750] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:19:48,753] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:19:56,086] INFO : VALIDATION | Step=1000/50000 | Train Loss=13.0955 | Valid Loss=13.7521 | Train Loss Unweighted=4.3589 | Valid Loss Unweighted=4.3445 | Stale=0 | ETA=361.5m +rank0 [2026-10-28 11:19:56,094] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:19:56,160] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:19:56,169] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:19:56,319] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:19:59,295] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:00,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:00,703] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:01,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:01,445] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-10-28 11:20:01,448] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:01,506] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:20:01,509] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:01,652] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:03,462] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:04,135] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:04,824] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:05,491] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:05,564] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:05,567] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:20:05,621] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:20:05,624] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:20:05,765] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:20:07,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:20:08,255] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:20:08,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:20:09,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:20:09,689] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:20:09,697] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:09,952] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:20:10,575] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:12,446] INFO : Plotted jets +rank0 [2026-10-28 11:20:13,537] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:13,569] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:13,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:20:14,169] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:15,540] INFO : Plotted jets +rank0 [2026-10-28 11:20:16,649] INFO : Plotted jet ratio +rank0 [2026-10-28 11:20:16,682] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:16,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_1000/clic_edm_qq_pf +rank0 [2026-10-28 11:20:17,296] INFO : Plotted number of elements +rank0 [2026-10-28 11:20:18,664] INFO : Plotted jets +rank0 [2026-10-28 11:20:19,752] INFO : Plotted jet ratio +rank0 [2026-10-28 11:21:00,512] INFO : Step 1100/50000 rank0 | Train Loss: 13.6489 | Train Loss Unweighted: 4.2287 | Train Loss Components: Classification: 0.0375 | Classification_binary: 3.7268 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2127 | Regression_eta: 0.0159 | Regression_pt: 0.2128 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5704 | Classification_binary: 3.7268 | Regression_cos_phi: 0.8141 | Regression_energy: 2.9414 | Regression_eta: 0.7824 | Regression_pt: 3.2019 | Regression_sin_phi: 0.6119 | LR: 9.99e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5259s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0867 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.2238 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0090 | Memory: rss_MB: 3210.7 | uss_MB: 2399.3 | child_rss_MB: 17792.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:21:00,737] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:21:39,790] INFO : Step 1200/50000 rank0 | Train Loss: 13.7197 | Train Loss Unweighted: 4.2789 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7803 | Regression_cos_phi: 0.0138 | Regression_energy: 0.2121 | Regression_eta: 0.0126 | Regression_pt: 0.2105 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4551 | Classification_binary: 3.7803 | Regression_cos_phi: 0.9219 | Regression_energy: 2.9335 | Regression_eta: 0.6202 | Regression_pt: 3.1672 | Regression_sin_phi: 0.8415 | LR: 9.99e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4191s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1852 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3210.5 | uss_MB: 2403.9 | child_rss_MB: 4105.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:21:40,003] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:19,313] INFO : Step 1300/50000 rank0 | Train Loss: 13.6193 | Train Loss Unweighted: 4.2909 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7996 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2108 | Regression_eta: 0.0134 | Regression_pt: 0.2038 | Regression_sin_phi: 0.0137 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.5366 | Classification_binary: 3.7996 | Regression_cos_phi: 0.8593 | Regression_energy: 2.9146 | Regression_eta: 0.6600 | Regression_pt: 3.0653 | Regression_sin_phi: 0.7839 | LR: 9.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5000s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2495 | max_optimizer_step: 0.1062 | max_tensorboard_logging: 0.4122 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 3210.5 | uss_MB: 2403.8 | child_rss_MB: 4103.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:22:19,525] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:22:59,245] INFO : Step 1400/50000 rank0 | Train Loss: 12.7212 | Train Loss Unweighted: 4.1945 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.7320 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1979 | Regression_eta: 0.0101 | Regression_pt: 0.1993 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3757 | Classification_binary: 3.7320 | Regression_cos_phi: 0.7072 | Regression_energy: 2.7364 | Regression_eta: 0.4961 | Regression_pt: 2.9987 | Regression_sin_phi: 0.6751 | LR: 9.98e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4799s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3079 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.3164 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3209.1 | uss_MB: 2402.5 | child_rss_MB: 4107.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:22:59,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:23:39,013] INFO : Step 1500/50000 rank0 | Train Loss: 12.6088 | Train Loss Unweighted: 4.0521 | Train Loss Components: Classification: 0.0348 | Classification_binary: 3.6019 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1892 | Regression_eta: 0.0135 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4581 | Classification_binary: 3.6019 | Regression_cos_phi: 0.7975 | Regression_energy: 2.6160 | Regression_eta: 0.6626 | Regression_pt: 2.8576 | Regression_sin_phi: 0.6151 | LR: 9.98e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4508s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1838 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3208.4 | uss_MB: 2402.5 | child_rss_MB: 4103.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:23:39,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:18,807] INFO : Step 1600/50000 rank0 | Train Loss: 13.0472 | Train Loss Unweighted: 4.0810 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.6044 | Regression_cos_phi: 0.0146 | Regression_energy: 0.2052 | Regression_eta: 0.0106 | Regression_pt: 0.2009 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3557 | Classification_binary: 3.6044 | Regression_cos_phi: 0.9764 | Regression_energy: 2.8382 | Regression_eta: 0.5187 | Regression_pt: 3.0224 | Regression_sin_phi: 0.7315 | LR: 9.98e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.4521s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0769 | max_loss: 0.0030 | max_backward: 0.3092 | max_optimizer_step: 0.1093 | max_tensorboard_logging: 0.3020 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3045.5 | uss_MB: 2240.2 | child_rss_MB: 4123.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:24:19,017] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:24:58,938] INFO : Step 1700/50000 rank0 | Train Loss: 12.1787 | Train Loss Unweighted: 4.0310 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.5923 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1886 | Regression_eta: 0.0093 | Regression_pt: 0.1857 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4319 | Classification_binary: 3.5923 | Regression_cos_phi: 0.7128 | Regression_energy: 2.6085 | Regression_eta: 0.4579 | Regression_pt: 2.7934 | Regression_sin_phi: 0.5818 | LR: 9.97e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4373s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0030 | max_backward: 0.3063 | max_optimizer_step: 0.1067 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3046.8 | uss_MB: 2242.6 | child_rss_MB: 4100.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:24:59,154] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:25:38,716] INFO : Step 1800/50000 rank0 | Train Loss: 12.8939 | Train Loss Unweighted: 4.0216 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.5469 | Regression_cos_phi: 0.0112 | Regression_energy: 0.2033 | Regression_eta: 0.0146 | Regression_pt: 0.2006 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4193 | Classification_binary: 3.5469 | Regression_cos_phi: 0.7500 | Regression_energy: 2.8110 | Regression_eta: 0.7166 | Regression_pt: 3.0179 | Regression_sin_phi: 0.6322 | LR: 9.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4712s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2801 | max_optimizer_step: 0.1032 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3046.5 | uss_MB: 2242.8 | child_rss_MB: 4097.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 11:25:38,927] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:18,902] INFO : Step 1900/50000 rank0 | Train Loss: 12.5651 | Train Loss Unweighted: 4.0999 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.6587 | Regression_cos_phi: 0.0129 | Regression_energy: 0.1818 | Regression_eta: 0.0113 | Regression_pt: 0.1885 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4473 | Classification_binary: 3.6587 | Regression_cos_phi: 0.8597 | Regression_energy: 2.5144 | Regression_eta: 0.5560 | Regression_pt: 2.8364 | Regression_sin_phi: 0.6927 | LR: 9.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5189s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3103 | max_optimizer_step: 0.1063 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3046.5 | uss_MB: 2242.9 | child_rss_MB: 4094.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:26:19,116] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:58,534] INFO : Step 2000/50000 rank0 | Train Loss: 12.3343 | Train Loss Unweighted: 3.9901 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.5403 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1918 | Regression_eta: 0.0149 | Regression_pt: 0.1916 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3712 | Classification_binary: 3.5403 | Regression_cos_phi: 0.5970 | Regression_energy: 2.6519 | Regression_eta: 0.7317 | Regression_pt: 2.8821 | Regression_sin_phi: 0.5600 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3413s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1300 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1030 | max_tensorboard_logging: 0.2759 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3046.5 | uss_MB: 2243.1 | child_rss_MB: 4095.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:26:58,747] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 153.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:26:58,790] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-2000.pth +rank0 [2026-10-28 11:26:58,960] INFO : Running validation on rank0 +rank0 [2026-10-28 11:26:58,960] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:26:58,960] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:27:00,253] INFO : saved batch0_epoch2000.parquet +rank0 [2026-10-28 11:27:00,394] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:27:00,522] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:27:00,595] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:27:00,663] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:27:00,731] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:27:00,800] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:27:00,868] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:27:00,976] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:27:01,105] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:27:01,235] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:27:01,305] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:27:01,373] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:27:01,439] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:27:01,508] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:27:01,577] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:27:01,680] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:27:01,688] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:27:01,690] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:27:01,693] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:27:01,694] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:27:01,697] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:27:01,699] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:27:01,700] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:27:01,702] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:27:01,704] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:27:01,706] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:27:01,709] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:27:01,712] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:27:01,713] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:27:01,715] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:27:01,718] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:27:09,162] INFO : VALIDATION | Step=2000/50000 | Train Loss=12.3343 | Valid Loss=12.5441 | Train Loss Unweighted=3.9901 | Valid Loss Unweighted=4.0316 | Stale=0 | ETA=350.3m +rank0 [2026-10-28 11:27:09,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:09,234] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:27:09,237] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:09,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:11,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:11,831] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:12,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:13,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:13,279] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:13,283] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:13,340] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:27:13,342] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:13,486] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:15,291] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:15,951] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:16,646] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:17,314] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:17,401] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:17,404] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:27:17,459] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:27:17,462] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:27:17,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:27:19,391] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:27:20,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:27:20,748] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:27:21,411] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:27:21,486] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:27:21,493] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:21,745] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:27:22,639] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:24,086] INFO : Plotted jets +rank0 [2026-10-28 11:27:25,168] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:25,200] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:25,425] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:27:25,794] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:27,171] INFO : Plotted jets +rank0 [2026-10-28 11:27:28,276] INFO : Plotted jet ratio +rank0 [2026-10-28 11:27:28,308] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:28,509] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_2000/clic_edm_qq_pf +rank0 [2026-10-28 11:27:28,916] INFO : Plotted number of elements +rank0 [2026-10-28 11:27:30,293] INFO : Plotted jets +rank0 [2026-10-28 11:27:31,790] INFO : Plotted jet ratio +rank0 [2026-10-28 11:28:12,271] INFO : Step 2100/50000 rank0 | Train Loss: 12.3404 | Train Loss Unweighted: 3.9792 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.5360 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1882 | Regression_eta: 0.0127 | Regression_pt: 0.1862 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.4525 | Classification_binary: 3.5360 | Regression_cos_phi: 0.6734 | Regression_energy: 2.6027 | Regression_eta: 0.6237 | Regression_pt: 2.8006 | Regression_sin_phi: 0.6515 | LR: 9.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4513s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1775 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.2868 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3250.6 | uss_MB: 2450.6 | child_rss_MB: 4046.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:28:12,483] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:28:52,054] INFO : Step 2200/50000 rank0 | Train Loss: 11.9485 | Train Loss Unweighted: 3.8666 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.4376 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1811 | Regression_eta: 0.0139 | Regression_pt: 0.1827 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2781 | Classification_binary: 3.4376 | Regression_cos_phi: 0.7587 | Regression_energy: 2.5041 | Regression_eta: 0.6822 | Regression_pt: 2.7482 | Regression_sin_phi: 0.5396 | LR: 9.96e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6120s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0774 | max_forward: 0.1050 | max_loss: 0.0031 | max_backward: 0.3954 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.2741 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3252.5 | uss_MB: 2452.7 | child_rss_MB: 4043.5 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:28:52,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:29:31,811] INFO : Step 2300/50000 rank0 | Train Loss: 12.1868 | Train Loss Unweighted: 3.9852 | Train Loss Components: Classification: 0.0319 | Classification_binary: 3.5422 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1910 | Regression_eta: 0.0104 | Regression_pt: 0.1882 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3374 | Classification_binary: 3.5422 | Regression_cos_phi: 0.6791 | Regression_energy: 2.6414 | Regression_eta: 0.5127 | Regression_pt: 2.8310 | Regression_sin_phi: 0.6431 | LR: 9.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0769 | max_loss: 0.0032 | max_backward: 0.3060 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.3929 | max_tensorboard_flush: 0.0147 | Memory: rss_MB: 3250.1 | uss_MB: 2450.7 | child_rss_MB: 4046.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:29:32,026] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:11,997] INFO : Step 2400/50000 rank0 | Train Loss: 11.9228 | Train Loss Unweighted: 3.8706 | Train Loss Components: Classification: 0.0310 | Classification_binary: 3.4490 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1796 | Regression_eta: 0.0144 | Regression_pt: 0.1747 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2989 | Classification_binary: 3.4490 | Regression_cos_phi: 0.6911 | Regression_energy: 2.4830 | Regression_eta: 0.7052 | Regression_pt: 2.6284 | Regression_sin_phi: 0.6672 | LR: 9.95e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3108 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3250.1 | uss_MB: 2450.7 | child_rss_MB: 4041.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:30:12,212] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:30:51,654] INFO : Step 2500/50000 rank0 | Train Loss: 11.5616 | Train Loss Unweighted: 3.9130 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.5057 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1728 | Regression_eta: 0.0102 | Regression_pt: 0.1723 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3692 | Classification_binary: 3.5057 | Regression_cos_phi: 0.6821 | Regression_energy: 2.3895 | Regression_eta: 0.4997 | Regression_pt: 2.5917 | Regression_sin_phi: 0.5238 | LR: 9.94e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4024s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1318 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1084 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3250.0 | uss_MB: 2450.7 | child_rss_MB: 4038.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:30:51,867] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:31:31,571] INFO : Step 2600/50000 rank0 | Train Loss: 12.4914 | Train Loss Unweighted: 3.9695 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.5127 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1947 | Regression_eta: 0.0131 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1800 | Classification_binary: 3.5127 | Regression_cos_phi: 0.8187 | Regression_energy: 2.6919 | Regression_eta: 0.6414 | Regression_pt: 2.9581 | Regression_sin_phi: 0.6887 | LR: 9.94e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.5475s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0031 | max_backward: 0.3076 | max_optimizer_step: 0.1113 | max_tensorboard_logging: 0.3912 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 3116.1 | uss_MB: 2316.8 | child_rss_MB: 4041.6 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:31:31,784] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:11,991] INFO : Step 2700/50000 rank0 | Train Loss: 12.6956 | Train Loss Unweighted: 3.9876 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.5223 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1977 | Regression_eta: 0.0128 | Regression_pt: 0.1993 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3184 | Classification_binary: 3.5223 | Regression_cos_phi: 0.8541 | Regression_energy: 2.7339 | Regression_eta: 0.6301 | Regression_pt: 2.9976 | Regression_sin_phi: 0.6390 | LR: 9.94e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.3991s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1406 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.2870 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3117.1 | uss_MB: 2318.9 | child_rss_MB: 4038.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:32:12,207] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:32:52,097] INFO : Step 2800/50000 rank0 | Train Loss: 11.9806 | Train Loss Unweighted: 3.8973 | Train Loss Components: Classification: 0.0313 | Classification_binary: 3.4799 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1748 | Regression_eta: 0.0169 | Regression_pt: 0.1727 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3093 | Classification_binary: 3.4799 | Regression_cos_phi: 0.7144 | Regression_energy: 2.4173 | Regression_eta: 0.8299 | Regression_pt: 2.5985 | Regression_sin_phi: 0.6313 | LR: 9.93e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4367s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1844 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1069 | max_tensorboard_logging: 0.2928 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3117.1 | uss_MB: 2319.0 | child_rss_MB: 4032.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:32:52,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:33:31,597] INFO : Step 2900/50000 rank0 | Train Loss: 11.6618 | Train Loss Unweighted: 3.7539 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.3450 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1726 | Regression_eta: 0.0156 | Regression_pt: 0.1676 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3257 | Classification_binary: 3.3450 | Regression_cos_phi: 0.6325 | Regression_energy: 2.3864 | Regression_eta: 0.7666 | Regression_pt: 2.5209 | Regression_sin_phi: 0.6846 | LR: 9.93e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4133s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1843 | max_loss: 0.0029 | max_backward: 0.2026 | max_optimizer_step: 0.1049 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3117.1 | uss_MB: 2319.1 | child_rss_MB: 4034.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:33:31,812] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:11,616] INFO : Step 3000/50000 rank0 | Train Loss: 11.7576 | Train Loss Unweighted: 3.9013 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.4849 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1769 | Regression_eta: 0.0121 | Regression_pt: 0.1753 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.3248 | Classification_binary: 3.4849 | Regression_cos_phi: 0.7113 | Regression_energy: 2.4456 | Regression_eta: 0.5950 | Regression_pt: 2.6380 | Regression_sin_phi: 0.5581 | LR: 9.92e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5207s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0030 | max_backward: 0.3094 | max_optimizer_step: 0.1065 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3117.1 | uss_MB: 2319.1 | child_rss_MB: 4034.5 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:34:11,828] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:34:11,846] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-3000.pth +rank0 [2026-10-28 11:34:12,119] INFO : Running validation on rank0 +rank0 [2026-10-28 11:34:12,119] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:34:12,119] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:34:13,298] INFO : saved batch0_epoch3000.parquet +rank0 [2026-10-28 11:34:13,441] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:34:13,570] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:34:13,644] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:34:13,711] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:34:13,778] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:34:13,846] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:34:13,913] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:34:14,019] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:34:14,148] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:34:14,279] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:34:14,348] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:34:14,415] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:34:14,481] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:34:14,549] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:34:14,617] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:34:14,718] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:34:14,725] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:34:14,728] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:34:14,731] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:34:14,732] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:34:14,735] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:34:14,737] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:34:14,740] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:34:14,741] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:34:14,848] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:34:14,851] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:34:14,852] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:34:14,855] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:34:14,858] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:34:14,861] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:34:14,863] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:34:22,272] INFO : VALIDATION | Step=3000/50000 | Train Loss=11.7576 | Valid Loss=11.7898 | Train Loss Unweighted=3.9013 | Valid Loss Unweighted=3.8000 | Stale=0 | ETA=341.7m +rank0 [2026-10-28 11:34:22,282] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:22,348] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:34:22,351] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:22,438] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:24,171] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:24,798] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:25,428] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:26,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:26,121] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:26,124] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:26,180] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:34:26,183] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:26,257] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:28,083] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:28,789] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:29,392] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:30,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:30,116] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:30,120] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:34:30,173] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:34:30,176] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:34:30,320] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:34:32,109] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:34:32,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:34:33,460] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:34:34,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:34:34,209] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:34:34,216] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:34,470] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:34:34,914] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:36,751] INFO : Plotted jets +rank0 [2026-10-28 11:34:37,826] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:37,857] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:38,081] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:34:38,448] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:39,813] INFO : Plotted jets +rank0 [2026-10-28 11:34:40,920] INFO : Plotted jet ratio +rank0 [2026-10-28 11:34:40,953] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:41,153] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_3000/clic_edm_qq_pf +rank0 [2026-10-28 11:34:41,561] INFO : Plotted number of elements +rank0 [2026-10-28 11:34:42,933] INFO : Plotted jets +rank0 [2026-10-28 11:34:44,405] INFO : Plotted jet ratio +rank0 [2026-10-28 11:35:24,177] INFO : Step 3100/50000 rank0 | Train Loss: 12.0916 | Train Loss Unweighted: 3.8176 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1935 | Regression_eta: 0.0103 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2855 | Classification_binary: 3.3700 | Regression_cos_phi: 0.7046 | Regression_energy: 2.6759 | Regression_eta: 0.5051 | Regression_pt: 2.8672 | Regression_sin_phi: 0.6833 | LR: 9.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4475s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1718 | max_loss: 0.0030 | max_backward: 0.2025 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.2892 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 3334.8 | uss_MB: 2538.7 | child_rss_MB: 4033.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:35:24,389] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:03,664] INFO : Step 3200/50000 rank0 | Train Loss: 11.2594 | Train Loss Unweighted: 3.8042 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.4140 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1631 | Regression_eta: 0.0132 | Regression_pt: 0.1644 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2337 | Classification_binary: 3.4140 | Regression_cos_phi: 0.6444 | Regression_energy: 2.2557 | Regression_eta: 0.6495 | Regression_pt: 2.4733 | Regression_sin_phi: 0.5887 | LR: 9.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4461s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1834 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.2820 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4034.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:36:03,877] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:36:42,471] INFO : Step 3300/50000 rank0 | Train Loss: 11.7776 | Train Loss Unweighted: 3.7029 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.2734 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1820 | Regression_eta: 0.0132 | Regression_pt: 0.1824 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2512 | Classification_binary: 3.2734 | Regression_cos_phi: 0.6102 | Regression_energy: 2.5172 | Regression_eta: 0.6465 | Regression_pt: 2.7445 | Regression_sin_phi: 0.7346 | LR: 9.90e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1134 | max_loss: 0.0033 | max_backward: 0.2023 | max_optimizer_step: 0.1109 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4032.0 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:36:42,683] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:37:22,527] INFO : Step 3400/50000 rank0 | Train Loss: 11.2672 | Train Loss Unweighted: 3.6481 | Train Loss Components: Classification: 0.0287 | Classification_binary: 3.2413 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1719 | Regression_eta: 0.0111 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2037 | Classification_binary: 3.2413 | Regression_cos_phi: 0.6797 | Regression_energy: 2.3771 | Regression_eta: 0.5458 | Regression_pt: 2.6235 | Regression_sin_phi: 0.5961 | LR: 9.90e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4825s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.2600 | max_optimizer_step: 0.1103 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3334.8 | uss_MB: 2538.8 | child_rss_MB: 4031.2 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:37:22,741] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:02,752] INFO : Step 3500/50000 rank0 | Train Loss: 11.5238 | Train Loss Unweighted: 3.7867 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.3813 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1689 | Regression_eta: 0.0140 | Regression_pt: 0.1719 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2443 | Classification_binary: 3.3813 | Regression_cos_phi: 0.6963 | Regression_energy: 2.3354 | Regression_eta: 0.6866 | Regression_pt: 2.5868 | Regression_sin_phi: 0.5931 | LR: 9.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6858s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1042 | max_loss: 0.0033 | max_backward: 0.3927 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3833 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3335.5 | uss_MB: 2540.9 | child_rss_MB: 4035.9 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:38:02,963] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:38:42,253] INFO : Step 3600/50000 rank0 | Train Loss: 10.5003 | Train Loss Unweighted: 3.5529 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.1695 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1663 | Regression_eta: 0.0089 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1620 | Classification_binary: 3.1695 | Regression_cos_phi: 0.5276 | Regression_energy: 2.3000 | Regression_eta: 0.4385 | Regression_pt: 2.4854 | Regression_sin_phi: 0.4173 | LR: 9.89e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5358s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2935 | max_optimizer_step: 0.1090 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3186.6 | uss_MB: 2390.1 | child_rss_MB: 4030.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:38:42,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:39:22,337] INFO : Step 3700/50000 rank0 | Train Loss: 10.5089 | Train Loss Unweighted: 3.6312 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.2631 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1564 | Regression_eta: 0.0101 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1986 | Classification_binary: 3.2631 | Regression_cos_phi: 0.5470 | Regression_energy: 2.1623 | Regression_eta: 0.4981 | Regression_pt: 2.3509 | Regression_sin_phi: 0.4889 | LR: 9.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5012s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3038 | max_optimizer_step: 0.1089 | max_tensorboard_logging: 0.3642 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3187.6 | uss_MB: 2391.6 | child_rss_MB: 4029.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:39:22,548] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:02,675] INFO : Step 3800/50000 rank0 | Train Loss: 10.7211 | Train Loss Unweighted: 3.6064 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.2434 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1502 | Regression_eta: 0.0133 | Regression_pt: 0.1501 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1555 | Classification_binary: 3.2434 | Regression_cos_phi: 0.5974 | Regression_energy: 2.0770 | Regression_eta: 0.6556 | Regression_pt: 2.2581 | Regression_sin_phi: 0.7340 | LR: 9.87e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4508s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0508 | max_backward: 0.2031 | max_optimizer_step: 0.1047 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 3187.6 | uss_MB: 2391.7 | child_rss_MB: 4027.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:40:02,889] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:40:42,444] INFO : Step 3900/50000 rank0 | Train Loss: 12.7464 | Train Loss Unweighted: 3.6417 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.1699 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1969 | Regression_eta: 0.0300 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2497 | Classification_binary: 3.1699 | Regression_cos_phi: 0.6527 | Regression_energy: 2.7235 | Regression_eta: 1.4724 | Regression_pt: 2.9484 | Regression_sin_phi: 0.5297 | LR: 9.87e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5703s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1862 | max_loss: 0.0030 | max_backward: 0.2869 | max_optimizer_step: 0.1699 | max_tensorboard_logging: 0.2800 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3187.8 | uss_MB: 2394.3 | child_rss_MB: 4029.2 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 11:40:42,667] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:21,968] INFO : Step 4000/50000 rank0 | Train Loss: 11.5654 | Train Loss Unweighted: 3.6366 | Train Loss Components: Classification: 0.0310 | Classification_binary: 3.2277 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1725 | Regression_eta: 0.0145 | Regression_pt: 0.1682 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2979 | Classification_binary: 3.2277 | Regression_cos_phi: 0.7732 | Regression_energy: 2.3861 | Regression_eta: 0.7121 | Regression_pt: 2.5305 | Regression_sin_phi: 0.6380 | LR: 9.86e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.4429s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0331 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.2040 | max_tensorboard_logging: 0.2797 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3187.6 | uss_MB: 2392.3 | child_rss_MB: 4025.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:41:22,195] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:41:22,214] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-4000.pth +rank0 [2026-10-28 11:41:22,386] INFO : Running validation on rank0 +rank0 [2026-10-28 11:41:22,386] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:41:22,386] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:41:23,700] INFO : saved batch0_epoch4000.parquet +rank0 [2026-10-28 11:41:23,843] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:41:23,972] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:41:24,044] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:41:24,111] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:41:24,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:41:24,247] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:41:24,316] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:41:24,425] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:41:24,554] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:41:24,682] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:41:24,752] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:41:24,819] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:41:24,887] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:41:24,956] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:41:25,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:41:25,136] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:41:25,142] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:41:25,145] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:41:25,148] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:41:25,149] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:41:25,152] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:41:25,154] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:41:25,155] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:41:25,157] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:41:25,160] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:41:25,161] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:41:25,163] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:41:25,166] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:41:25,167] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:41:25,169] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:41:25,172] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:41:32,615] INFO : VALIDATION | Step=4000/50000 | Train Loss=11.5654 | Valid Loss=11.1201 | Train Loss Unweighted=3.6366 | Valid Loss Unweighted=3.6235 | Stale=0 | ETA=333.3m +rank0 [2026-10-28 11:41:32,626] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:32,693] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:41:32,696] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:32,842] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:34,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:35,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:35,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:36,666] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:36,741] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:41:36,744] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:36,799] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:41:36,802] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:36,878] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:38,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:39,470] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:40,079] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:40,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:40,815] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:41:40,819] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:41:40,873] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:41:40,876] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:41:41,023] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:41:42,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:41:43,473] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:41:44,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:41:44,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:41:44,916] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:41:44,923] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:45,176] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:41:45,624] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:47,461] INFO : Plotted jets +rank0 [2026-10-28 11:41:48,550] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:48,582] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:48,803] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:41:49,177] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:50,550] INFO : Plotted jets +rank0 [2026-10-28 11:41:51,661] INFO : Plotted jet ratio +rank0 [2026-10-28 11:41:51,693] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:41:51,894] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_4000/clic_edm_qq_pf +rank0 [2026-10-28 11:41:52,309] INFO : Plotted number of elements +rank0 [2026-10-28 11:41:53,691] INFO : Plotted jets +rank0 [2026-10-28 11:41:55,161] INFO : Plotted jet ratio +rank0 [2026-10-28 11:42:34,831] INFO : Step 4100/50000 rank0 | Train Loss: 11.0707 | Train Loss Unweighted: 3.6203 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.2302 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1647 | Regression_eta: 0.0122 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2511 | Classification_binary: 3.2302 | Regression_cos_phi: 0.6834 | Regression_energy: 2.2772 | Regression_eta: 0.6017 | Regression_pt: 2.4532 | Regression_sin_phi: 0.5739 | LR: 9.85e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1280 | max_loss: 0.0034 | max_backward: 0.2028 | max_optimizer_step: 0.1105 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4020.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:42:35,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:14,956] INFO : Step 4200/50000 rank0 | Train Loss: 10.5140 | Train Loss Unweighted: 3.5173 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.1305 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1676 | Regression_eta: 0.0093 | Regression_pt: 0.1672 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1625 | Classification_binary: 3.1305 | Regression_cos_phi: 0.5208 | Regression_energy: 2.3172 | Regression_eta: 0.4581 | Regression_pt: 2.5147 | Regression_sin_phi: 0.4102 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5795s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0031 | max_backward: 0.3060 | max_optimizer_step: 0.1160 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4021.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:43:15,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:43:55,324] INFO : Step 4300/50000 rank0 | Train Loss: 10.4742 | Train Loss Unweighted: 3.4542 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.0921 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1526 | Regression_eta: 0.0118 | Regression_pt: 0.1493 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1382 | Classification_binary: 3.0921 | Regression_cos_phi: 0.6938 | Regression_energy: 2.1098 | Regression_eta: 0.5806 | Regression_pt: 2.2467 | Regression_sin_phi: 0.6129 | LR: 9.84e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5300s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0824 | max_backward: 0.2032 | max_optimizer_step: 0.1112 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4018.9 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:43:55,538] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:44:35,056] INFO : Step 4400/50000 rank0 | Train Loss: 10.1726 | Train Loss Unweighted: 3.4727 | Train Loss Components: Classification: 0.0255 | Classification_binary: 3.1148 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1536 | Regression_eta: 0.0117 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0674 | Classification_binary: 3.1148 | Regression_cos_phi: 0.5871 | Regression_energy: 2.1243 | Regression_eta: 0.5743 | Regression_pt: 2.2673 | Regression_sin_phi: 0.4375 | LR: 9.83e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5413s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1001 | max_forward: 0.0777 | max_loss: 0.0030 | max_backward: 0.2031 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4018.1 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:44:35,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:14,872] INFO : Step 4500/50000 rank0 | Train Loss: 11.7776 | Train Loss Unweighted: 3.6103 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.1637 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1918 | Regression_eta: 0.0155 | Regression_pt: 0.1924 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2614 | Classification_binary: 3.1637 | Regression_cos_phi: 0.5703 | Regression_energy: 2.6517 | Regression_eta: 0.7615 | Regression_pt: 2.8945 | Regression_sin_phi: 0.4745 | LR: 9.82e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5795s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0032 | max_backward: 0.3062 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3014 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3421.7 | uss_MB: 2630.0 | child_rss_MB: 4049.8 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:45:15,083] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:45:55,956] INFO : Step 4600/50000 rank0 | Train Loss: 11.5346 | Train Loss Unweighted: 3.5693 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.1461 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1791 | Regression_eta: 0.0131 | Regression_pt: 0.1802 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.2191 | Classification_binary: 3.1461 | Regression_cos_phi: 0.6960 | Regression_energy: 2.4769 | Regression_eta: 0.6457 | Regression_pt: 2.7108 | Regression_sin_phi: 0.6400 | LR: 9.81e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1048 | max_loss: 0.0033 | max_backward: 0.3195 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3926 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3421.7 | uss_MB: 2632.1 | child_rss_MB: 4016.0 | allocated_MB: 295.1 | reserved_MB: 3778.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 11:45:56,168] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:46:36,275] INFO : Step 4700/50000 rank0 | Train Loss: 10.5234 | Train Loss Unweighted: 3.5185 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1449 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1607 | Regression_eta: 0.0106 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1388 | Classification_binary: 3.1449 | Regression_cos_phi: 0.5906 | Regression_energy: 2.2222 | Regression_eta: 0.5190 | Regression_pt: 2.3577 | Regression_sin_phi: 0.5504 | LR: 9.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4174s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1067 | max_loss: 0.0033 | max_backward: 0.2027 | max_optimizer_step: 0.1095 | max_tensorboard_logging: 0.3057 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3258.1 | uss_MB: 2467.6 | child_rss_MB: 4013.7 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:46:36,491] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:16,258] INFO : Step 4800/50000 rank0 | Train Loss: 10.3396 | Train Loss Unweighted: 3.4477 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1525 | Regression_eta: 0.0111 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1254 | Classification_binary: 3.0859 | Regression_cos_phi: 0.6664 | Regression_energy: 2.1090 | Regression_eta: 0.5432 | Regression_pt: 2.2920 | Regression_sin_phi: 0.5178 | LR: 9.80e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5667s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.3080 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0122 | Memory: rss_MB: 3258.1 | uss_MB: 2467.6 | child_rss_MB: 4015.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:47:16,470] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:47:55,232] INFO : Step 4900/50000 rank0 | Train Loss: 10.0105 | Train Loss Unweighted: 3.3531 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.0015 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1492 | Regression_eta: 0.0101 | Regression_pt: 0.1481 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0722 | Classification_binary: 3.0015 | Regression_cos_phi: 0.6114 | Regression_energy: 2.0634 | Regression_eta: 0.4969 | Regression_pt: 2.2287 | Regression_sin_phi: 0.5364 | LR: 9.79e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4504s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1799 | max_loss: 0.0030 | max_backward: 0.2027 | max_optimizer_step: 0.1082 | max_tensorboard_logging: 0.2819 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3258.1 | uss_MB: 2468.1 | child_rss_MB: 4016.3 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:47:55,453] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:36,174] INFO : Step 5000/50000 rank0 | Train Loss: 10.7358 | Train Loss Unweighted: 3.5550 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.1712 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1633 | Regression_eta: 0.0108 | Regression_pt: 0.1641 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0774 | Classification_binary: 3.1712 | Regression_cos_phi: 0.6698 | Regression_energy: 2.2584 | Regression_eta: 0.5286 | Regression_pt: 2.4695 | Regression_sin_phi: 0.5610 | LR: 9.78e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.3998s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.1310 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1079 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 3258.1 | uss_MB: 2468.1 | child_rss_MB: 4022.4 | allocated_MB: 291.4 | reserved_MB: 3778.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:48:36,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:48:36,408] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-5000.pth +rank0 [2026-10-28 11:48:36,581] INFO : Running validation on rank0 +rank0 [2026-10-28 11:48:36,582] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:48:36,582] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:48:37,864] INFO : saved batch0_epoch5000.parquet +rank0 [2026-10-28 11:48:38,008] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:48:38,136] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:48:38,208] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:48:38,275] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:48:38,342] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:48:38,411] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:48:38,479] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:48:38,590] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:48:38,719] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:48:38,847] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:48:38,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:48:38,984] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:48:39,050] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:48:39,118] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:48:39,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:48:39,296] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:48:39,304] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:48:39,307] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:48:39,310] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:48:39,311] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:48:39,314] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:48:39,317] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:48:39,320] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:48:39,321] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:48:39,323] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:48:39,326] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:48:39,326] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:48:39,330] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:48:39,332] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:48:39,335] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:48:39,336] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:48:46,637] INFO : VALIDATION | Step=5000/50000 | Train Loss=10.7358 | Valid Loss=10.7371 | Train Loss Unweighted=3.5550 | Valid Loss Unweighted=3.4633 | Stale=0 | ETA=326.0m +rank0 [2026-10-28 11:48:46,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:46,710] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:48:46,714] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:46,779] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:48,673] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:49,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:50,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:50,797] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:50,870] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:48:50,874] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:50,936] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:48:50,938] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:48:51,090] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:52,886] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:53,559] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:54,249] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:54,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:55,019] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:48:55,022] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:48:55,083] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:48:55,086] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:48:55,237] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:48:57,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:48:57,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:48:58,395] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:48:59,062] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:48:59,145] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:48:59,153] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:59,412] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:48:59,869] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:01,747] INFO : Plotted jets +rank0 [2026-10-28 11:49:02,840] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:02,875] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:03,103] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:49:03,476] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:04,848] INFO : Plotted jets +rank0 [2026-10-28 11:49:05,968] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:06,002] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:06,204] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_5000/clic_edm_qq_pf +rank0 [2026-10-28 11:49:06,621] INFO : Plotted number of elements +rank0 [2026-10-28 11:49:08,001] INFO : Plotted jets +rank0 [2026-10-28 11:49:09,111] INFO : Plotted jet ratio +rank0 [2026-10-28 11:49:49,518] INFO : Step 5100/50000 rank0 | Train Loss: 10.5352 | Train Loss Unweighted: 3.4065 | Train Loss Components: Classification: 0.0242 | Classification_binary: 3.0227 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1655 | Regression_eta: 0.0102 | Regression_pt: 0.1639 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0138 | Classification_binary: 3.0227 | Regression_cos_phi: 0.6897 | Regression_energy: 2.2890 | Regression_eta: 0.5021 | Regression_pt: 2.4650 | Regression_sin_phi: 0.5529 | LR: 9.77e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2651 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.3949 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4019.0 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 11:49:49,730] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:50:29,472] INFO : Step 5200/50000 rank0 | Train Loss: 10.6893 | Train Loss Unweighted: 3.4241 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.0325 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1678 | Regression_eta: 0.0111 | Regression_pt: 0.1678 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0718 | Classification_binary: 3.0325 | Regression_cos_phi: 0.6188 | Regression_energy: 2.3204 | Regression_eta: 0.5440 | Regression_pt: 2.5251 | Regression_sin_phi: 0.5767 | LR: 9.76e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.4695s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3094 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.2929 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4041.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:50:29,685] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:08,793] INFO : Step 5300/50000 rank0 | Train Loss: 9.7687 | Train Loss Unweighted: 3.4334 | Train Loss Components: Classification: 0.0234 | Classification_binary: 3.0874 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1493 | Regression_eta: 0.0093 | Regression_pt: 0.1488 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9793 | Classification_binary: 3.0874 | Regression_cos_phi: 0.4698 | Regression_energy: 2.0643 | Regression_eta: 0.4546 | Regression_pt: 2.2381 | Regression_sin_phi: 0.4753 | LR: 9.75e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5943s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1036 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2027 | max_optimizer_step: 0.1810 | max_tensorboard_logging: 0.3393 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4015.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:51:09,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:51:48,275] INFO : Step 5400/50000 rank0 | Train Loss: 10.4975 | Train Loss Unweighted: 3.4566 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.0878 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1565 | Regression_eta: 0.0109 | Regression_pt: 0.1541 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1208 | Classification_binary: 3.0878 | Regression_cos_phi: 0.6981 | Regression_energy: 2.1643 | Regression_eta: 0.5348 | Regression_pt: 2.3183 | Regression_sin_phi: 0.5734 | LR: 9.74e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5148s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0235 | max_forward: 0.0780 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1847 | max_tensorboard_logging: 0.3155 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3483.1 | uss_MB: 2693.5 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:51:48,490] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:52:28,801] INFO : Step 5500/50000 rank0 | Train Loss: 10.6236 | Train Loss Unweighted: 3.2562 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.8804 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1579 | Regression_eta: 0.0176 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1114 | Classification_binary: 2.8804 | Regression_cos_phi: 0.7031 | Regression_energy: 2.1833 | Regression_eta: 0.8632 | Regression_pt: 2.3055 | Regression_sin_phi: 0.5768 | LR: 9.73e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5820s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3033 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3966 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3483.1 | uss_MB: 2693.6 | child_rss_MB: 4017.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:52:29,018] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:07,705] INFO : Step 5600/50000 rank0 | Train Loss: 10.1159 | Train Loss Unweighted: 3.3475 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.9753 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1630 | Regression_eta: 0.0116 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9548 | Classification_binary: 2.9753 | Regression_cos_phi: 0.5776 | Regression_energy: 2.2540 | Regression_eta: 0.5712 | Regression_pt: 2.3983 | Regression_sin_phi: 0.3848 | LR: 9.72e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5410s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0445 | max_backward: 0.2026 | max_optimizer_step: 0.1120 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3483.1 | uss_MB: 2693.6 | child_rss_MB: 4012.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:53:07,918] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:53:48,458] INFO : Step 5700/50000 rank0 | Train Loss: 10.1841 | Train Loss Unweighted: 3.4032 | Train Loss Components: Classification: 0.0237 | Classification_binary: 3.0420 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1563 | Regression_eta: 0.0122 | Regression_pt: 0.1511 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9941 | Classification_binary: 3.0420 | Regression_cos_phi: 0.6416 | Regression_energy: 2.1615 | Regression_eta: 0.6015 | Regression_pt: 2.2733 | Regression_sin_phi: 0.4701 | LR: 9.71e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3095 | max_optimizer_step: 0.1156 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3331.3 | uss_MB: 2541.7 | child_rss_MB: 4012.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:53:48,673] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:54:28,582] INFO : Step 5800/50000 rank0 | Train Loss: 10.2215 | Train Loss Unweighted: 3.3291 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.9586 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1607 | Regression_eta: 0.0096 | Regression_pt: 0.1569 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9765 | Classification_binary: 2.9586 | Regression_cos_phi: 0.6407 | Regression_energy: 2.2223 | Regression_eta: 0.4706 | Regression_pt: 2.3598 | Regression_sin_phi: 0.5930 | LR: 9.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5301s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.3093 | max_optimizer_step: 0.1099 | max_tensorboard_logging: 0.3779 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3331.3 | uss_MB: 2541.7 | child_rss_MB: 4012.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:54:28,795] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:08,834] INFO : Step 5900/50000 rank0 | Train Loss: 10.3740 | Train Loss Unweighted: 3.2798 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.9110 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1561 | Regression_eta: 0.0118 | Regression_pt: 0.1528 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1644 | Classification_binary: 2.9110 | Regression_cos_phi: 0.6861 | Regression_energy: 2.1586 | Regression_eta: 0.5816 | Regression_pt: 2.2988 | Regression_sin_phi: 0.5735 | LR: 9.69e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2825 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.3815 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3331.3 | uss_MB: 2541.9 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:55:09,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:48,775] INFO : Step 6000/50000 rank0 | Train Loss: 10.4734 | Train Loss Unweighted: 3.3621 | Train Loss Components: Classification: 0.0262 | Classification_binary: 2.9774 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1660 | Regression_eta: 0.0087 | Regression_pt: 0.1645 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0966 | Classification_binary: 2.9774 | Regression_cos_phi: 0.6844 | Regression_energy: 2.2952 | Regression_eta: 0.4285 | Regression_pt: 2.4748 | Regression_sin_phi: 0.5164 | LR: 9.68e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.4426s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.3070 | max_optimizer_step: 0.1081 | max_tensorboard_logging: 0.2899 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3331.3 | uss_MB: 2542.0 | child_rss_MB: 4033.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:55:48,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:55:49,006] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-6000.pth +rank0 [2026-10-28 11:55:49,179] INFO : Running validation on rank0 +rank0 [2026-10-28 11:55:49,179] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 11:55:49,179] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 11:55:50,420] INFO : saved batch0_epoch6000.parquet +rank0 [2026-10-28 11:55:50,564] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 11:55:50,693] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 11:55:50,765] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 11:55:50,832] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 11:55:50,900] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 11:55:50,969] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 11:55:51,036] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 11:55:51,146] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 11:55:51,274] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 11:55:51,403] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 11:55:51,473] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 11:55:51,541] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 11:55:51,609] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 11:55:51,677] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 11:55:51,745] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 11:55:51,845] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 11:55:51,852] INFO : plotted pt_target histogram +rank0 [2026-10-28 11:55:51,854] INFO : plotted pt_pred histogram +rank0 [2026-10-28 11:55:51,857] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 11:55:51,858] INFO : plotted eta_target histogram +rank0 [2026-10-28 11:55:51,861] INFO : plotted eta_pred histogram +rank0 [2026-10-28 11:55:51,864] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 11:55:51,864] INFO : plotted sphi_target histogram +rank0 [2026-10-28 11:55:51,867] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 11:55:51,870] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 11:55:51,870] INFO : plotted cphi_target histogram +rank0 [2026-10-28 11:55:51,873] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 11:55:51,876] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 11:55:51,876] INFO : plotted energy_target histogram +rank0 [2026-10-28 11:55:51,879] INFO : plotted energy_pred histogram +rank0 [2026-10-28 11:55:51,880] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 11:55:59,263] INFO : VALIDATION | Step=6000/50000 | Train Loss=10.4734 | Valid Loss=10.2158 | Train Loss Unweighted=3.3621 | Valid Loss Unweighted=3.3282 | Stale=0 | ETA=318.5m +rank0 [2026-10-28 11:55:59,273] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:55:59,340] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 11:55:59,344] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 11:55:59,489] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:01,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:01,970] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:02,681] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:03,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:03,471] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 11:56:03,474] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:56:03,539] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 11:56:03,542] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:03,611] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:05,463] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:06,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:06,725] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:07,394] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:07,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:56:07,481] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 11:56:07,543] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 11:56:07,546] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 11:56:07,696] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 11:56:09,480] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 11:56:10,159] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 11:56:10,857] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 11:56:11,525] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 11:56:11,607] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 11:56:11,615] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:56:11,869] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ttbar_pf +rank0 [2026-10-28 11:56:12,318] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:14,218] INFO : Plotted jets +rank0 [2026-10-28 11:56:15,307] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:15,341] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:15,565] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 11:56:15,937] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:17,313] INFO : Plotted jets +rank0 [2026-10-28 11:56:18,431] INFO : Plotted jet ratio +rank0 [2026-10-28 11:56:18,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:56:18,735] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_6000/clic_edm_qq_pf +rank0 [2026-10-28 11:56:19,148] INFO : Plotted number of elements +rank0 [2026-10-28 11:56:20,532] INFO : Plotted jets +rank0 [2026-10-28 11:56:21,625] INFO : Plotted jet ratio +rank0 [2026-10-28 11:57:01,411] INFO : Step 6100/50000 rank0 | Train Loss: 10.0922 | Train Loss Unweighted: 3.3107 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.9494 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1536 | Regression_eta: 0.0093 | Regression_pt: 0.1539 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0638 | Classification_binary: 2.9494 | Regression_cos_phi: 0.6051 | Regression_energy: 2.1238 | Regression_eta: 0.4584 | Regression_pt: 2.3150 | Regression_sin_phi: 0.5767 | LR: 9.67e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5148s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0544 | max_backward: 0.2029 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3547.6 | uss_MB: 2761.4 | child_rss_MB: 4010.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:57:01,625] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:57:41,438] INFO : Step 6200/50000 rank0 | Train Loss: 9.9202 | Train Loss Unweighted: 3.2990 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.9514 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1459 | Regression_eta: 0.0109 | Regression_pt: 0.1474 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0145 | Classification_binary: 2.9514 | Regression_cos_phi: 0.6173 | Regression_energy: 2.0178 | Regression_eta: 0.5345 | Regression_pt: 2.2171 | Regression_sin_phi: 0.5676 | LR: 9.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5720s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.3083 | max_optimizer_step: 0.1137 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:57:41,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:58:21,192] INFO : Step 6300/50000 rank0 | Train Loss: 9.5818 | Train Loss Unweighted: 3.3361 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9944 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1467 | Regression_eta: 0.0082 | Regression_pt: 0.1487 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.9944 | Regression_cos_phi: 0.5355 | Regression_energy: 2.0288 | Regression_eta: 0.4053 | Regression_pt: 2.2366 | Regression_sin_phi: 0.4650 | LR: 9.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5246s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0741 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.2032 | max_optimizer_step: 0.1820 | max_tensorboard_logging: 0.2945 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4004.7 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:58:21,419] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:00,640] INFO : Step 6400/50000 rank0 | Train Loss: 10.5963 | Train Loss Unweighted: 3.3863 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.9966 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1681 | Regression_eta: 0.0120 | Regression_pt: 0.1663 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0558 | Classification_binary: 2.9966 | Regression_cos_phi: 0.6322 | Regression_energy: 2.3243 | Regression_eta: 0.5901 | Regression_pt: 2.5013 | Regression_sin_phi: 0.4961 | LR: 9.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4715s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0537 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1879 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 11:59:00,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 11:59:40,499] INFO : Step 6500/50000 rank0 | Train Loss: 10.2834 | Train Loss Unweighted: 3.2737 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8982 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1617 | Regression_eta: 0.0121 | Regression_pt: 0.1592 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9700 | Classification_binary: 2.8982 | Regression_cos_phi: 0.6373 | Regression_energy: 2.2364 | Regression_eta: 0.5952 | Regression_pt: 2.3953 | Regression_sin_phi: 0.5509 | LR: 9.63e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5580s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0921 | max_backward: 0.2027 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4003.3 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 11:59:40,713] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:00:21,097] INFO : Step 6600/50000 rank0 | Train Loss: 10.0484 | Train Loss Unweighted: 3.2922 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9384 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1493 | Regression_eta: 0.0108 | Regression_pt: 0.1494 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9873 | Classification_binary: 2.9384 | Regression_cos_phi: 0.6692 | Regression_energy: 2.0646 | Regression_eta: 0.5322 | Regression_pt: 2.2483 | Regression_sin_phi: 0.6084 | LR: 9.62e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4999s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3084 | max_optimizer_step: 0.1148 | max_tensorboard_logging: 0.2922 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3547.6 | uss_MB: 2761.5 | child_rss_MB: 4031.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:00:21,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:00,933] INFO : Step 6700/50000 rank0 | Train Loss: 10.1948 | Train Loss Unweighted: 3.3072 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.9468 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1526 | Regression_eta: 0.0124 | Regression_pt: 0.1517 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9672 | Classification_binary: 2.9468 | Regression_cos_phi: 0.7079 | Regression_energy: 2.1107 | Regression_eta: 0.6072 | Regression_pt: 2.2825 | Regression_sin_phi: 0.5724 | LR: 9.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4953s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0031 | max_backward: 0.2269 | max_optimizer_step: 0.1117 | max_tensorboard_logging: 0.3904 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3402.6 | uss_MB: 2615.4 | child_rss_MB: 4007.8 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:01:01,148] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:01:40,577] INFO : Step 6800/50000 rank0 | Train Loss: 9.6889 | Train Loss Unweighted: 3.2773 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.9353 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1459 | Regression_eta: 0.0107 | Regression_pt: 0.1456 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9472 | Classification_binary: 2.9353 | Regression_cos_phi: 0.6053 | Regression_energy: 2.0170 | Regression_eta: 0.5248 | Regression_pt: 2.1903 | Regression_sin_phi: 0.4690 | LR: 9.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3508s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0011 | max_forward: 0.1028 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1079 | max_tensorboard_logging: 0.2773 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3402.6 | uss_MB: 2615.5 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:01:40,793] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 152.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:02:19,961] INFO : Step 6900/50000 rank0 | Train Loss: 9.3526 | Train Loss Unweighted: 3.1332 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.7952 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1465 | Regression_eta: 0.0078 | Regression_pt: 0.1443 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9650 | Classification_binary: 2.7952 | Regression_cos_phi: 0.5339 | Regression_energy: 2.0264 | Regression_eta: 0.3810 | Regression_pt: 2.1708 | Regression_sin_phi: 0.4804 | LR: 9.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5016s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0567 | max_backward: 0.2025 | max_optimizer_step: 0.1101 | max_tensorboard_logging: 0.3816 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3402.6 | uss_MB: 2615.6 | child_rss_MB: 4006.4 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:02:20,173] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:00,160] INFO : Step 7000/50000 rank0 | Train Loss: 10.1169 | Train Loss Unweighted: 3.2794 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.9182 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1545 | Regression_eta: 0.0110 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9900 | Classification_binary: 2.9182 | Regression_cos_phi: 0.6860 | Regression_energy: 2.1371 | Regression_eta: 0.5420 | Regression_pt: 2.2857 | Regression_sin_phi: 0.5580 | LR: 9.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4417s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1820 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1095 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3402.6 | uss_MB: 2615.7 | child_rss_MB: 4007.2 | allocated_MB: 291.4 | reserved_MB: 3650.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:03:00,375] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:03:00,410] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-7000.pth +rank0 [2026-10-28 12:03:00,589] INFO : Running validation on rank0 +rank0 [2026-10-28 12:03:00,589] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:03:00,589] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:03:01,892] INFO : saved batch0_epoch7000.parquet +rank0 [2026-10-28 12:03:02,034] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:03:02,164] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:03:02,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:03:02,303] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:03:02,370] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:03:02,439] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:03:02,506] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:03:02,618] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:03:02,746] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:03:02,875] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:03:02,950] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:03:03,018] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:03:03,085] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:03:03,154] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:03:03,222] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:03:03,323] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:03:03,329] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:03:03,332] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:03:03,333] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:03:03,336] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:03:03,337] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:03:03,339] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:03:03,342] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:03:03,343] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:03:03,345] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:03:03,348] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:03:03,349] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:03:03,351] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:03:03,354] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:03:03,355] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:03:03,357] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:03:10,814] INFO : VALIDATION | Step=7000/50000 | Train Loss=10.1169 | Valid Loss=9.9681 | Train Loss Unweighted=3.2794 | Valid Loss Unweighted=3.2478 | Stale=0 | ETA=311.0m +rank0 [2026-10-28 12:03:10,824] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:10,884] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:03:10,887] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:11,034] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:12,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:13,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:14,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:15,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:15,149] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:03:15,152] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:15,210] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:03:15,212] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:15,293] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:17,142] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:17,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:18,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:19,140] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:19,223] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:03:19,226] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:03:19,291] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:03:19,294] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:03:19,381] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:03:21,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:03:21,838] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:03:22,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:03:23,190] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:03:23,273] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:03:23,280] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:23,534] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:03:23,980] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:25,834] INFO : Plotted jets +rank0 [2026-10-28 12:03:26,918] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:26,950] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:27,174] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:03:27,544] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:28,906] INFO : Plotted jets +rank0 [2026-10-28 12:03:30,013] INFO : Plotted jet ratio +rank0 [2026-10-28 12:03:30,045] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 12:03:30,241] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_7000/clic_edm_qq_pf +rank0 [2026-10-28 12:03:30,650] INFO : Plotted number of elements +rank0 [2026-10-28 12:03:32,044] INFO : Plotted jets +rank0 [2026-10-28 12:03:33,131] INFO : Plotted jet ratio +rank0 [2026-10-28 12:04:12,724] INFO : Step 7100/50000 rank0 | Train Loss: 10.0366 | Train Loss Unweighted: 3.2548 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8943 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1535 | Regression_eta: 0.0096 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0569 | Classification_binary: 2.8943 | Regression_cos_phi: 0.6837 | Regression_energy: 2.1222 | Regression_eta: 0.4707 | Regression_pt: 2.3034 | Regression_sin_phi: 0.5054 | LR: 9.56e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5407s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2787 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3851 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4008.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:12,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:04:52,271] INFO : Step 7200/50000 rank0 | Train Loss: 9.8051 | Train Loss Unweighted: 3.3297 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.9756 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1530 | Regression_eta: 0.0093 | Regression_pt: 0.1531 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9114 | Classification_binary: 2.9756 | Regression_cos_phi: 0.4991 | Regression_energy: 2.1155 | Regression_eta: 0.4568 | Regression_pt: 2.3030 | Regression_sin_phi: 0.5437 | LR: 9.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1663 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.2909 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:04:52,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:05:31,503] INFO : Step 7300/50000 rank0 | Train Loss: 10.0799 | Train Loss Unweighted: 3.3576 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.9919 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1575 | Regression_eta: 0.0084 | Regression_pt: 0.1571 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0559 | Classification_binary: 2.9919 | Regression_cos_phi: 0.5677 | Regression_energy: 2.1781 | Regression_eta: 0.4134 | Regression_pt: 2.3638 | Regression_sin_phi: 0.5091 | LR: 9.53e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4971s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.2245 | max_optimizer_step: 0.1133 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:05:31,715] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:10,955] INFO : Step 7400/50000 rank0 | Train Loss: 10.3337 | Train Loss Unweighted: 3.2562 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8856 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1589 | Regression_eta: 0.0106 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9747 | Classification_binary: 2.8856 | Regression_cos_phi: 0.7083 | Regression_energy: 2.1969 | Regression_eta: 0.5223 | Regression_pt: 2.3264 | Regression_sin_phi: 0.7196 | LR: 9.52e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5363s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0032 | max_backward: 0.2692 | max_optimizer_step: 0.1146 | max_tensorboard_logging: 0.3852 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4009.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:11,171] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:06:51,091] INFO : Step 7500/50000 rank0 | Train Loss: 10.1096 | Train Loss Unweighted: 3.2043 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8364 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1604 | Regression_eta: 0.0110 | Regression_pt: 0.1533 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9760 | Classification_binary: 2.8364 | Regression_cos_phi: 0.6698 | Regression_energy: 2.2183 | Regression_eta: 0.5417 | Regression_pt: 2.3062 | Regression_sin_phi: 0.5612 | LR: 9.51e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5191s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2671 | max_optimizer_step: 0.1136 | max_tensorboard_logging: 0.3754 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3617.7 | uss_MB: 2833.9 | child_rss_MB: 4012.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:06:51,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:07:30,759] INFO : Step 7600/50000 rank0 | Train Loss: 9.5990 | Train Loss Unweighted: 3.1519 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.8134 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1443 | Regression_eta: 0.0105 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9177 | Classification_binary: 2.8134 | Regression_cos_phi: 0.6387 | Regression_energy: 1.9948 | Regression_eta: 0.5147 | Regression_pt: 2.1377 | Regression_sin_phi: 0.5819 | LR: 9.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0030 | max_backward: 0.2027 | max_optimizer_step: 0.1158 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3617.7 | uss_MB: 2834.0 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:07:30,973] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:10,588] INFO : Step 7700/50000 rank0 | Train Loss: 9.9504 | Train Loss Unweighted: 3.2750 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.9115 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1574 | Regression_eta: 0.0096 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9373 | Classification_binary: 2.9115 | Regression_cos_phi: 0.5974 | Regression_energy: 2.1772 | Regression_eta: 0.4711 | Regression_pt: 2.3503 | Regression_sin_phi: 0.5056 | LR: 9.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4617s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0164 | max_forward: 0.0778 | max_loss: 0.0031 | max_backward: 0.2031 | max_optimizer_step: 0.1846 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3477.2 | uss_MB: 2691.9 | child_rss_MB: 4007.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:08:10,817] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:08:50,550] INFO : Step 7800/50000 rank0 | Train Loss: 10.2935 | Train Loss Unweighted: 3.2838 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.9211 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1529 | Regression_eta: 0.0129 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0429 | Classification_binary: 2.9211 | Regression_cos_phi: 0.7729 | Regression_energy: 2.1140 | Regression_eta: 0.6363 | Regression_pt: 2.2717 | Regression_sin_phi: 0.5347 | LR: 9.47e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4623s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1838 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1098 | max_tensorboard_logging: 0.2997 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3477.2 | uss_MB: 2692.0 | child_rss_MB: 4007.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:08:50,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:09:30,739] INFO : Step 7900/50000 rank0 | Train Loss: 10.5627 | Train Loss Unweighted: 3.2527 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1671 | Regression_eta: 0.0106 | Regression_pt: 0.1661 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9675 | Classification_binary: 2.8631 | Regression_cos_phi: 0.7461 | Regression_energy: 2.3113 | Regression_eta: 0.5228 | Regression_pt: 2.4988 | Regression_sin_phi: 0.6531 | LR: 9.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0495 | max_backward: 0.2035 | max_optimizer_step: 0.1080 | max_tensorboard_logging: 0.3721 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3477.2 | uss_MB: 2692.2 | child_rss_MB: 4012.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:09:30,953] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:10:10,968] INFO : Step 8000/50000 rank0 | Train Loss: 9.7712 | Train Loss Unweighted: 3.2109 | Train Loss Components: Classification: 0.0258 | Classification_binary: 2.8568 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1534 | Regression_eta: 0.0096 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0807 | Classification_binary: 2.8568 | Regression_cos_phi: 0.5271 | Regression_energy: 2.1208 | Regression_eta: 0.4720 | Regression_pt: 2.2444 | Regression_sin_phi: 0.4694 | LR: 9.44e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5226s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0770 | max_loss: 0.0938 | max_backward: 0.2027 | max_optimizer_step: 0.1108 | max_tensorboard_logging: 0.3768 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3477.2 | uss_MB: 2692.2 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:10:11,179] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:10:11,250] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-8000.pth +rank0 [2026-10-28 12:10:11,425] INFO : Running validation on rank0 +rank0 [2026-10-28 12:10:11,425] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:10:11,425] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:10:12,716] INFO : saved batch0_epoch8000.parquet +rank0 [2026-10-28 12:10:12,860] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:10:12,989] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:10:13,061] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:10:13,129] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:10:13,196] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:10:13,264] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:10:13,332] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:10:13,442] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:10:13,571] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:10:13,700] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:10:13,769] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:10:13,837] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:10:13,904] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:10:13,972] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:10:14,041] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:10:14,158] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:10:14,165] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:10:14,168] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:10:14,171] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:10:14,172] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:10:14,175] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:10:14,177] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:10:14,178] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:10:14,181] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:10:14,183] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:10:14,184] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:10:14,187] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:10:14,189] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:10:14,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:10:14,193] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:10:14,195] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:10:21,628] INFO : VALIDATION | Step=8000/50000 | Train Loss=9.7712 | Valid Loss=9.9230 | Train Loss Unweighted=3.2109 | Valid Loss Unweighted=3.2146 | Stale=0 | ETA=303.5m +rank0 [2026-10-28 12:10:21,637] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:21,699] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:10:21,702] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:21,851] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:23,623] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:24,292] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:24,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:25,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:25,748] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:25,751] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:25,807] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:10:25,810] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:25,909] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:27,705] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:28,389] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:29,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:29,749] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:29,840] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:10:29,843] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:10:29,896] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:10:29,899] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:10:30,053] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:10:31,812] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:10:32,488] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:10:33,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:10:33,836] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:10:33,924] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:10:33,931] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:34,183] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:10:34,630] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:36,498] INFO : Plotted jets +rank0 [2026-10-28 12:10:37,599] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:37,633] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:37,853] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:10:38,226] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:39,597] INFO : Plotted jets +rank0 [2026-10-28 12:10:40,653] INFO : Plotted jet ratio +rank0 [2026-10-28 12:10:40,688] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 12:10:40,890] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_8000/clic_edm_qq_pf +rank0 [2026-10-28 12:10:41,302] INFO : Plotted number of elements +rank0 [2026-10-28 12:10:42,679] INFO : Plotted jets +rank0 [2026-10-28 12:10:43,764] INFO : Plotted jet ratio +rank0 [2026-10-28 12:11:23,948] INFO : Step 8100/50000 rank0 | Train Loss: 9.6271 | Train Loss Unweighted: 3.1567 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.8186 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1419 | Regression_eta: 0.0098 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9971 | Classification_binary: 2.8186 | Regression_cos_phi: 0.6290 | Regression_energy: 1.9623 | Regression_eta: 0.4817 | Regression_pt: 2.1477 | Regression_sin_phi: 0.5905 | LR: 9.43e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5881s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3097 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.3823 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3711.3 | uss_MB: 2926.8 | child_rss_MB: 4006.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:11:24,163] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 48.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:04,051] INFO : Step 8200/50000 rank0 | Train Loss: 9.4943 | Train Loss Unweighted: 3.2026 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.8705 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1393 | Regression_eta: 0.0102 | Regression_pt: 0.1420 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9443 | Classification_binary: 2.8705 | Regression_cos_phi: 0.5838 | Regression_energy: 1.9267 | Regression_eta: 0.4989 | Regression_pt: 2.1356 | Regression_sin_phi: 0.5345 | LR: 9.42e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5575s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3078 | max_optimizer_step: 0.1139 | max_tensorboard_logging: 0.3713 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3711.3 | uss_MB: 2926.8 | child_rss_MB: 4008.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:12:04,265] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:12:44,118] INFO : Step 8300/50000 rank0 | Train Loss: 9.6151 | Train Loss Unweighted: 3.1340 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.7764 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1550 | Regression_eta: 0.0096 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9320 | Classification_binary: 2.7764 | Regression_cos_phi: 0.5352 | Regression_energy: 2.1440 | Regression_eta: 0.4718 | Regression_pt: 2.3369 | Regression_sin_phi: 0.4188 | LR: 9.40e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5646s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.3080 | max_optimizer_step: 0.1164 | max_tensorboard_logging: 0.2789 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:12:44,331] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:13:23,716] INFO : Step 8400/50000 rank0 | Train Loss: 9.9661 | Train Loss Unweighted: 3.1275 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7585 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1602 | Regression_eta: 0.0109 | Regression_pt: 0.1570 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9107 | Classification_binary: 2.7585 | Regression_cos_phi: 0.6547 | Regression_energy: 2.2157 | Regression_eta: 0.5365 | Regression_pt: 2.3614 | Regression_sin_phi: 0.5286 | LR: 9.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5304s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.2631 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.3831 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4005.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:13:23,928] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:03,834] INFO : Step 8500/50000 rank0 | Train Loss: 9.6554 | Train Loss Unweighted: 3.1441 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.8051 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1423 | Regression_eta: 0.0100 | Regression_pt: 0.1422 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0492 | Classification_binary: 2.8051 | Regression_cos_phi: 0.6497 | Regression_energy: 1.9676 | Regression_eta: 0.4899 | Regression_pt: 2.1397 | Regression_sin_phi: 0.5542 | LR: 9.37e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6039s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3109 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3985 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4011.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:04,047] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:14:43,930] INFO : Step 8600/50000 rank0 | Train Loss: 9.4448 | Train Loss Unweighted: 3.1686 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.8290 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1458 | Regression_eta: 0.0090 | Regression_pt: 0.1459 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9319 | Classification_binary: 2.8290 | Regression_cos_phi: 0.5732 | Regression_energy: 2.0166 | Regression_eta: 0.4444 | Regression_pt: 2.1954 | Regression_sin_phi: 0.4542 | LR: 9.36e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0336 | max_backward: 0.2034 | max_optimizer_step: 0.1152 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3712.3 | uss_MB: 2926.9 | child_rss_MB: 4009.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:14:44,142] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:15:24,217] INFO : Step 8700/50000 rank0 | Train Loss: 9.1874 | Train Loss Unweighted: 2.9863 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.6668 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1338 | Regression_eta: 0.0098 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9925 | Classification_binary: 2.6668 | Regression_cos_phi: 0.6248 | Regression_energy: 1.8497 | Regression_eta: 0.4811 | Regression_pt: 1.9970 | Regression_sin_phi: 0.5755 | LR: 9.34e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.2652 | max_optimizer_step: 0.1123 | max_tensorboard_logging: 0.3870 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3553.4 | uss_MB: 2767.8 | child_rss_MB: 4010.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:15:24,430] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:04,526] INFO : Step 8800/50000 rank0 | Train Loss: 10.2198 | Train Loss Unweighted: 3.2391 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.8778 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1509 | Regression_eta: 0.0131 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0200 | Classification_binary: 2.8778 | Regression_cos_phi: 0.7283 | Regression_energy: 2.0869 | Regression_eta: 0.6432 | Regression_pt: 2.2839 | Regression_sin_phi: 0.5797 | LR: 9.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1227 | max_loss: 0.0031 | max_backward: 0.2032 | max_optimizer_step: 0.1114 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 3553.4 | uss_MB: 2768.0 | child_rss_MB: 4003.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:16:04,740] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:16:43,871] INFO : Step 8900/50000 rank0 | Train Loss: 10.0898 | Train Loss Unweighted: 3.1272 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7541 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1605 | Regression_eta: 0.0100 | Regression_pt: 0.1583 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0216 | Classification_binary: 2.7541 | Regression_cos_phi: 0.5974 | Regression_energy: 2.2188 | Regression_eta: 0.4937 | Regression_pt: 2.3813 | Regression_sin_phi: 0.6229 | LR: 9.31e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0792 | max_backward: 0.2029 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.3807 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3553.4 | uss_MB: 2768.2 | child_rss_MB: 4002.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:16:44,086] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:23,203] INFO : Step 9000/50000 rank0 | Train Loss: 9.0820 | Train Loss Unweighted: 3.0574 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7379 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1351 | Regression_eta: 0.0093 | Regression_pt: 0.1359 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9470 | Classification_binary: 2.7379 | Regression_cos_phi: 0.5842 | Regression_energy: 1.8682 | Regression_eta: 0.4586 | Regression_pt: 2.0451 | Regression_sin_phi: 0.4411 | LR: 9.30e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5917s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0032 | max_backward: 0.3113 | max_optimizer_step: 0.1130 | max_tensorboard_logging: 0.4039 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3553.4 | uss_MB: 2768.0 | child_rss_MB: 4005.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:17:23,416] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:17:23,440] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-9000.pth +rank0 [2026-10-28 12:17:23,721] INFO : Running validation on rank0 +rank0 [2026-10-28 12:17:23,721] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:17:23,721] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:17:24,906] INFO : saved batch0_epoch9000.parquet +rank0 [2026-10-28 12:17:25,049] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:17:25,181] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:17:25,254] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:17:25,322] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:17:25,390] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:17:25,458] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:17:25,526] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:17:25,637] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:17:25,768] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:17:25,899] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:17:25,970] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:17:26,038] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:17:26,105] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:17:26,172] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:17:26,240] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:17:26,344] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:17:26,449] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:17:26,452] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:17:26,454] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:17:26,457] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:17:26,458] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:17:26,461] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:17:26,464] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:17:26,465] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:17:26,467] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:17:26,468] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:17:26,471] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:17:26,473] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:17:26,474] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:17:26,477] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:17:26,479] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:17:33,808] INFO : VALIDATION | Step=9000/50000 | Train Loss=9.0820 | Valid Loss=9.7088 | Train Loss Unweighted=3.0574 | Valid Loss Unweighted=3.1556 | Stale=0 | ETA=296.1m +rank0 [2026-10-28 12:17:33,818] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:33,880] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:17:33,884] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:34,072] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:36,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:36,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:37,398] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:38,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:38,164] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:17:38,167] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:38,267] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:17:38,286] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:38,442] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:40,168] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:40,825] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:41,516] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:42,198] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:42,284] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:17:42,287] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:17:42,365] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:17:42,368] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:17:42,522] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:17:44,282] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:17:44,935] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:17:45,624] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:17:46,299] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:17:46,393] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:17:46,400] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:46,656] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:17:47,103] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:48,991] INFO : Plotted jets +rank0 [2026-10-28 12:17:50,087] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:50,120] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:50,345] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:17:50,717] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:52,089] INFO : Plotted jets +rank0 [2026-10-28 12:17:53,204] INFO : Plotted jet ratio +rank0 [2026-10-28 12:17:53,238] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:53,442] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_9000/clic_edm_qq_pf +rank0 [2026-10-28 12:17:53,865] INFO : Plotted number of elements +rank0 [2026-10-28 12:17:55,251] INFO : Plotted jets +rank0 [2026-10-28 12:17:56,350] INFO : Plotted jet ratio +rank0 [2026-10-28 12:18:36,809] INFO : Step 9100/50000 rank0 | Train Loss: 9.4462 | Train Loss Unweighted: 3.1075 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7681 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1459 | Regression_eta: 0.0099 | Regression_pt: 0.1442 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9149 | Classification_binary: 2.7681 | Regression_cos_phi: 0.5806 | Regression_energy: 2.0180 | Regression_eta: 0.4850 | Regression_pt: 2.1696 | Regression_sin_phi: 0.5099 | LR: 9.28e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5157s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1865 | max_loss: 0.0032 | max_backward: 0.2034 | max_optimizer_step: 0.1155 | max_tensorboard_logging: 0.3126 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3624.4 | uss_MB: 2838.1 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:18:37,031] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:16,738] INFO : Step 9200/50000 rank0 | Train Loss: 9.6513 | Train Loss Unweighted: 3.0969 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7428 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1519 | Regression_eta: 0.0098 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9394 | Classification_binary: 2.7428 | Regression_cos_phi: 0.5857 | Regression_energy: 2.1003 | Regression_eta: 0.4818 | Regression_pt: 2.2911 | Regression_sin_phi: 0.5103 | LR: 9.27e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5686s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3045 | max_optimizer_step: 0.1176 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3626.4 | uss_MB: 2839.5 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:19:16,950] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:19:56,763] INFO : Step 9300/50000 rank0 | Train Loss: 8.9964 | Train Loss Unweighted: 3.0644 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.7522 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1330 | Regression_eta: 0.0093 | Regression_pt: 0.1309 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9222 | Classification_binary: 2.7522 | Regression_cos_phi: 0.5767 | Regression_energy: 1.8386 | Regression_eta: 0.4574 | Regression_pt: 1.9691 | Regression_sin_phi: 0.4803 | LR: 9.25e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4845s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3050 | max_optimizer_step: 0.1218 | max_tensorboard_logging: 0.2727 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3627.4 | uss_MB: 2839.7 | child_rss_MB: 4022.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:19:56,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:20:37,252] INFO : Step 9400/50000 rank0 | Train Loss: 9.4310 | Train Loss Unweighted: 3.1401 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.8116 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1394 | Regression_eta: 0.0100 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9731 | Classification_binary: 2.8116 | Regression_cos_phi: 0.6445 | Regression_energy: 1.9273 | Regression_eta: 0.4931 | Regression_pt: 2.0622 | Regression_sin_phi: 0.5193 | LR: 9.24e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5755s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0031 | max_backward: 0.3097 | max_optimizer_step: 0.1162 | max_tensorboard_logging: 0.3924 | max_tensorboard_flush: 0.0073 | Memory: rss_MB: 3627.4 | uss_MB: 2839.8 | child_rss_MB: 4000.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:20:37,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:17,040] INFO : Step 9500/50000 rank0 | Train Loss: 9.5144 | Train Loss Unweighted: 3.1366 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7996 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1445 | Regression_eta: 0.0101 | Regression_pt: 0.1410 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9606 | Classification_binary: 2.7996 | Regression_cos_phi: 0.5773 | Regression_energy: 1.9982 | Regression_eta: 0.4975 | Regression_pt: 2.1212 | Regression_sin_phi: 0.5600 | LR: 9.22e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5028s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0429 | max_forward: 0.0779 | max_loss: 0.0031 | max_backward: 0.2035 | max_optimizer_step: 0.1839 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3627.4 | uss_MB: 2839.9 | child_rss_MB: 4000.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:21:17,268] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:21:56,949] INFO : Step 9600/50000 rank0 | Train Loss: 10.0864 | Train Loss Unweighted: 3.1032 | Train Loss Components: Classification: 0.0243 | Classification_binary: 2.7340 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1577 | Regression_eta: 0.0105 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0197 | Classification_binary: 2.7340 | Regression_cos_phi: 0.7017 | Regression_energy: 2.1807 | Regression_eta: 0.5170 | Regression_pt: 2.3444 | Regression_sin_phi: 0.5890 | LR: 9.21e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4876s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1840 | max_loss: 0.0032 | max_backward: 0.2030 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2937 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 3627.4 | uss_MB: 2839.9 | child_rss_MB: 4003.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:21:57,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:22:37,905] INFO : Step 9700/50000 rank0 | Train Loss: 9.7767 | Train Loss Unweighted: 3.1829 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.8360 | Regression_cos_phi: 0.0109 | Regression_energy: 0.1450 | Regression_eta: 0.0104 | Regression_pt: 0.1489 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9771 | Classification_binary: 2.8360 | Regression_cos_phi: 0.7263 | Regression_energy: 2.0049 | Regression_eta: 0.5088 | Regression_pt: 2.2406 | Regression_sin_phi: 0.4830 | LR: 9.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5840s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0714 | max_backward: 0.2029 | max_optimizer_step: 0.1141 | max_tensorboard_logging: 0.4343 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3626.5 | uss_MB: 2838.0 | child_rss_MB: 4003.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:22:38,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:17,637] INFO : Step 9800/50000 rank0 | Train Loss: 10.3028 | Train Loss Unweighted: 3.2484 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8711 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1620 | Regression_eta: 0.0110 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 2.8711 | Regression_cos_phi: 0.6186 | Regression_energy: 2.2396 | Regression_eta: 0.5396 | Regression_pt: 2.4138 | Regression_sin_phi: 0.6289 | LR: 9.17e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.4714s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3066 | max_optimizer_step: 0.1122 | max_tensorboard_logging: 0.2851 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3627.5 | uss_MB: 2839.5 | child_rss_MB: 4032.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:23:17,849] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:23:57,269] INFO : Step 9900/50000 rank0 | Train Loss: 9.7808 | Train Loss Unweighted: 3.1641 | Train Loss Components: Classification: 0.0240 | Classification_binary: 2.8155 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1475 | Regression_eta: 0.0101 | Regression_pt: 0.1477 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0065 | Classification_binary: 2.8155 | Regression_cos_phi: 0.6915 | Regression_energy: 2.0391 | Regression_eta: 0.4976 | Regression_pt: 2.2217 | Regression_sin_phi: 0.5089 | LR: 9.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0034 | max_backward: 0.3087 | max_optimizer_step: 0.1124 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3627.5 | uss_MB: 2839.7 | child_rss_MB: 4006.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:23:57,482] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:37,108] INFO : Step 10000/50000 rank0 | Train Loss: 11.4549 | Train Loss Unweighted: 3.3507 | Train Loss Components: Classification: 0.0263 | Classification_binary: 2.9295 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1760 | Regression_eta: 0.0176 | Regression_pt: 0.1776 | Regression_sin_phi: 0.0128 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.1037 | Classification_binary: 2.9295 | Regression_cos_phi: 0.7199 | Regression_energy: 2.4339 | Regression_eta: 0.8671 | Regression_pt: 2.6723 | Regression_sin_phi: 0.7285 | LR: 9.14e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6342s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.2059 | max_loss: 0.0031 | max_backward: 0.2871 | max_optimizer_step: 0.1877 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3627.5 | uss_MB: 2842.2 | child_rss_MB: 4009.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:24:37,335] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:24:37,354] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-10000.pth +rank0 [2026-10-28 12:24:37,525] INFO : Running validation on rank0 +rank0 [2026-10-28 12:24:37,525] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:24:37,525] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:24:38,798] INFO : saved batch0_epoch10000.parquet +rank0 [2026-10-28 12:24:38,942] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:24:39,073] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:24:39,147] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:24:39,215] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:24:39,283] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:24:39,353] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:24:39,422] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:24:39,532] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:24:39,660] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:24:39,790] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:24:39,860] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:24:39,928] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:24:39,995] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:24:40,064] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:24:40,133] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:24:40,235] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:24:40,242] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:24:40,245] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:24:40,248] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:24:40,249] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:24:40,251] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:24:40,254] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:24:40,255] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:24:40,258] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:24:40,259] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:24:40,261] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:24:40,264] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:24:40,265] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:24:40,268] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:24:40,271] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:24:40,272] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:24:47,561] INFO : VALIDATION | Step=10000/50000 | Train Loss=11.4549 | Valid Loss=9.6757 | Train Loss Unweighted=3.3507 | Valid Loss Unweighted=3.1405 | Stale=0 | ETA=288.9m +rank0 [2026-10-28 12:24:47,570] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:47,661] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:24:47,664] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:24:47,813] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:49,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:50,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:51,057] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:51,763] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:51,872] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:24:51,876] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:51,932] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:24:51,935] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:24:52,021] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:53,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:54,554] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:55,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:55,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:24:55,887] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:24:55,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:24:55,970] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:24:55,973] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:24:56,131] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:24:57,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:24:58,628] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:24:59,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:24:59,931] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:25:00,025] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:25:00,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:25:00,282] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:25:00,727] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:02,632] INFO : Plotted jets +rank0 [2026-10-28 12:25:03,717] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:03,750] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:03,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:25:04,365] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:05,734] INFO : Plotted jets +rank0 [2026-10-28 12:25:06,842] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:06,875] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 12:25:07,074] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_10000/clic_edm_qq_pf +rank0 [2026-10-28 12:25:07,483] INFO : Plotted number of elements +rank0 [2026-10-28 12:25:08,863] INFO : Plotted jets +rank0 [2026-10-28 12:25:09,949] INFO : Plotted jet ratio +rank0 [2026-10-28 12:25:50,559] INFO : Step 10100/50000 rank0 | Train Loss: 10.1513 | Train Loss Unweighted: 3.2501 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.8889 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1526 | Regression_eta: 0.0120 | Regression_pt: 0.1523 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9938 | Classification_binary: 2.8889 | Regression_cos_phi: 0.6849 | Regression_energy: 2.1099 | Regression_eta: 0.5900 | Regression_pt: 2.2906 | Regression_sin_phi: 0.5933 | LR: 9.12e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1854 | max_loss: 0.0032 | max_backward: 0.2029 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2995 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3700.5 | uss_MB: 2912.6 | child_rss_MB: 4002.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:25:50,777] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:26:29,892] INFO : Step 10200/50000 rank0 | Train Loss: 9.4393 | Train Loss Unweighted: 3.0790 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7480 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1401 | Regression_eta: 0.0107 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9575 | Classification_binary: 2.7480 | Regression_cos_phi: 0.6749 | Regression_energy: 1.9367 | Regression_eta: 0.5279 | Regression_pt: 2.0791 | Regression_sin_phi: 0.5152 | LR: 9.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4883s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1142 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 3700.5 | uss_MB: 2913.7 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:26:30,110] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:09,559] INFO : Step 10300/50000 rank0 | Train Loss: 9.2976 | Train Loss Unweighted: 3.1214 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.7827 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1455 | Regression_eta: 0.0081 | Regression_pt: 0.1471 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9449 | Classification_binary: 2.7827 | Regression_cos_phi: 0.4745 | Regression_energy: 2.0122 | Regression_eta: 0.3993 | Regression_pt: 2.2129 | Regression_sin_phi: 0.4710 | LR: 9.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5581s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0911 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1774 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 3701.5 | uss_MB: 2914.0 | child_rss_MB: 4006.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:27:09,781] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:27:49,400] INFO : Step 10400/50000 rank0 | Train Loss: 9.9020 | Train Loss Unweighted: 3.1716 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.8254 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1487 | Regression_eta: 0.0127 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9523 | Classification_binary: 2.8254 | Regression_cos_phi: 0.7748 | Regression_energy: 2.0568 | Regression_eta: 0.6248 | Regression_pt: 2.1174 | Regression_sin_phi: 0.5505 | LR: 9.07e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4032s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1093 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2864 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 3701.5 | uss_MB: 2914.2 | child_rss_MB: 4007.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:27:49,615] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:28:29,708] INFO : Step 10500/50000 rank0 | Train Loss: 9.2844 | Train Loss Unweighted: 3.0615 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7291 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1428 | Regression_eta: 0.0090 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9802 | Classification_binary: 2.7291 | Regression_cos_phi: 0.6084 | Regression_energy: 1.9752 | Regression_eta: 0.4427 | Regression_pt: 2.1118 | Regression_sin_phi: 0.4370 | LR: 9.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5190s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0989 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1792 | max_tensorboard_logging: 0.2736 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3701.5 | uss_MB: 2914.3 | child_rss_MB: 4005.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:28:29,933] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:09,316] INFO : Step 10600/50000 rank0 | Train Loss: 8.9772 | Train Loss Unweighted: 3.0284 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7074 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1383 | Regression_eta: 0.0094 | Regression_pt: 0.1371 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8634 | Classification_binary: 2.7074 | Regression_cos_phi: 0.5437 | Regression_energy: 1.9123 | Regression_eta: 0.4636 | Regression_pt: 2.0630 | Regression_sin_phi: 0.4239 | LR: 9.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0475 | max_forward: 0.0777 | max_loss: 0.0031 | max_backward: 0.2026 | max_optimizer_step: 0.1790 | max_tensorboard_logging: 0.2681 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3701.5 | uss_MB: 2914.3 | child_rss_MB: 4001.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:29:09,545] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:29:49,495] INFO : Step 10700/50000 rank0 | Train Loss: 8.6186 | Train Loss Unweighted: 2.9534 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.6556 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1277 | Regression_eta: 0.0111 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8309 | Classification_binary: 2.6556 | Regression_cos_phi: 0.5287 | Regression_energy: 1.7657 | Regression_eta: 0.5458 | Regression_pt: 1.8607 | Regression_sin_phi: 0.4311 | LR: 9.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4867s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.3101 | max_optimizer_step: 0.1167 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3699.6 | uss_MB: 2912.3 | child_rss_MB: 4023.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 12:29:49,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:30:28,880] INFO : Step 10800/50000 rank0 | Train Loss: 9.1274 | Train Loss Unweighted: 2.9811 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.6519 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1405 | Regression_eta: 0.0082 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9284 | Classification_binary: 2.6519 | Regression_cos_phi: 0.5314 | Regression_energy: 1.9423 | Regression_eta: 0.4039 | Regression_pt: 2.1178 | Regression_sin_phi: 0.5516 | LR: 9.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3940s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1312 | max_loss: 0.0031 | max_backward: 0.2025 | max_optimizer_step: 0.1107 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3700.6 | uss_MB: 2913.9 | child_rss_MB: 4004.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:30:29,094] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 151.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:08,706] INFO : Step 10900/50000 rank0 | Train Loss: 9.3713 | Train Loss Unweighted: 3.0842 | Train Loss Components: Classification: 0.0238 | Classification_binary: 2.7456 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1457 | Regression_eta: 0.0097 | Regression_pt: 0.1437 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9973 | Classification_binary: 2.7456 | Regression_cos_phi: 0.5253 | Regression_energy: 2.0142 | Regression_eta: 0.4780 | Regression_pt: 2.1620 | Regression_sin_phi: 0.4489 | LR: 8.99e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.4331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0035 | max_backward: 0.3093 | max_optimizer_step: 0.1126 | max_tensorboard_logging: 0.2736 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3701.6 | uss_MB: 2914.2 | child_rss_MB: 4026.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 12:31:08,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:48,890] INFO : Step 11000/50000 rank0 | Train Loss: 9.4699 | Train Loss Unweighted: 3.0787 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7430 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1429 | Regression_eta: 0.0107 | Regression_pt: 0.1408 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9585 | Classification_binary: 2.7430 | Regression_cos_phi: 0.6423 | Regression_energy: 1.9757 | Regression_eta: 0.5270 | Regression_pt: 2.1175 | Regression_sin_phi: 0.5059 | LR: 8.97e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.4522s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0031 | max_backward: 0.3103 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.2734 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3701.6 | uss_MB: 2914.2 | child_rss_MB: 4003.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 12:31:49,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:31:49,122] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-11000.pth +rank0 [2026-10-28 12:31:49,301] INFO : Running validation on rank0 +rank0 [2026-10-28 12:31:49,301] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:31:49,301] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:31:50,572] INFO : saved batch0_epoch11000.parquet +rank0 [2026-10-28 12:31:50,714] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:31:50,844] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:31:50,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:31:50,984] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:31:51,052] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:31:51,121] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:31:51,189] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:31:51,300] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:31:51,428] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:31:51,558] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:31:51,627] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:31:51,696] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:31:51,763] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:31:51,832] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:31:51,901] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:31:52,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:31:52,018] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:31:52,021] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:31:52,024] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:31:52,025] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:31:52,027] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:31:52,030] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:31:52,031] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:31:52,033] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:31:52,036] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:31:52,037] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:31:52,039] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:31:52,042] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:31:52,043] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:31:52,045] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:31:52,048] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:31:59,426] INFO : VALIDATION | Step=11000/50000 | Train Loss=9.4699 | Valid Loss=9.5823 | Train Loss Unweighted=3.0787 | Valid Loss Unweighted=3.1123 | Stale=0 | ETA=281.6m +rank0 [2026-10-28 12:31:59,437] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:31:59,522] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:31:59,525] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:31:59,677] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:01,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:02,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:02,840] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:03,555] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:03,650] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:32:03,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:03,741] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:32:03,758] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:03,915] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:05,663] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:06,365] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:06,973] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:07,621] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:07,718] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:07,722] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:32:07,794] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:32:07,797] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:32:07,954] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:32:09,691] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:32:10,372] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:32:11,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:32:11,718] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:32:11,816] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:32:11,823] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:32:12,075] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:32:12,523] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:14,440] INFO : Plotted jets +rank0 [2026-10-28 12:32:15,529] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:15,562] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:15,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:32:16,160] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:17,528] INFO : Plotted jets +rank0 [2026-10-28 12:32:18,641] INFO : Plotted jet ratio +rank0 [2026-10-28 12:32:18,674] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 12:32:18,879] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_11000/clic_edm_qq_pf +rank0 [2026-10-28 12:32:19,289] INFO : Plotted number of elements +rank0 [2026-10-28 12:32:20,674] INFO : Plotted jets +rank0 [2026-10-28 12:32:21,764] INFO : Plotted jet ratio +rank0 [2026-10-28 12:33:02,364] INFO : Step 11100/50000 rank0 | Train Loss: 9.3667 | Train Loss Unweighted: 3.0498 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7210 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1408 | Regression_eta: 0.0100 | Regression_pt: 0.1360 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9418 | Classification_binary: 2.7210 | Regression_cos_phi: 0.7342 | Regression_energy: 1.9469 | Regression_eta: 0.4929 | Regression_pt: 2.0465 | Regression_sin_phi: 0.4834 | LR: 8.95e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.4930s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0550 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.2032 | max_optimizer_step: 0.1684 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3773.1 | uss_MB: 2987.9 | child_rss_MB: 4003.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:02,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:33:42,728] INFO : Step 11200/50000 rank0 | Train Loss: 9.3205 | Train Loss Unweighted: 3.0531 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.7267 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1381 | Regression_eta: 0.0108 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0345 | Classification_binary: 2.7267 | Regression_cos_phi: 0.5827 | Regression_energy: 1.9093 | Regression_eta: 0.5295 | Regression_pt: 2.0352 | Regression_sin_phi: 0.5026 | LR: 8.93e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5572s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3054 | max_optimizer_step: 0.1163 | max_tensorboard_logging: 0.2766 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3774.1 | uss_MB: 2989.0 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:33:42,940] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:34:23,138] INFO : Step 11300/50000 rank0 | Train Loss: 9.4410 | Train Loss Unweighted: 3.0109 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6734 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1457 | Regression_eta: 0.0124 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9131 | Classification_binary: 2.6734 | Regression_cos_phi: 0.6360 | Regression_energy: 2.0152 | Regression_eta: 0.6094 | Regression_pt: 2.0951 | Regression_sin_phi: 0.4988 | LR: 8.91e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0491 | max_backward: 0.2030 | max_optimizer_step: 0.1165 | max_tensorboard_logging: 0.3654 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3774.1 | uss_MB: 2989.2 | child_rss_MB: 4002.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:34:23,353] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:02,789] INFO : Step 11400/50000 rank0 | Train Loss: 8.9424 | Train Loss Unweighted: 3.0045 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.7113 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1192 | Regression_eta: 0.0126 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8943 | Classification_binary: 2.7113 | Regression_cos_phi: 0.6945 | Regression_energy: 1.6489 | Regression_eta: 0.6203 | Regression_pt: 1.7963 | Regression_sin_phi: 0.5768 | LR: 8.89e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1754 | max_loss: 0.0031 | max_backward: 0.2024 | max_optimizer_step: 0.1161 | max_tensorboard_logging: 0.2758 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3774.1 | uss_MB: 2989.3 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:35:03,006] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:35:42,875] INFO : Step 11500/50000 rank0 | Train Loss: 9.8073 | Train Loss Unweighted: 2.9903 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.6407 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1473 | Regression_eta: 0.0158 | Regression_pt: 0.1438 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9572 | Classification_binary: 2.6407 | Regression_cos_phi: 0.6854 | Regression_energy: 2.0375 | Regression_eta: 0.7761 | Regression_pt: 2.1633 | Regression_sin_phi: 0.5471 | LR: 8.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5447s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.1078 | max_backward: 0.2032 | max_optimizer_step: 0.1157 | max_tensorboard_logging: 0.3587 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3774.1 | uss_MB: 2989.3 | child_rss_MB: 4005.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:35:43,089] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:36:22,868] INFO : Step 11600/50000 rank0 | Train Loss: 9.5090 | Train Loss Unweighted: 3.0012 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.6543 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1483 | Regression_eta: 0.0109 | Regression_pt: 0.1466 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9943 | Classification_binary: 2.6543 | Regression_cos_phi: 0.5675 | Regression_energy: 2.0509 | Regression_eta: 0.5337 | Regression_pt: 2.2061 | Regression_sin_phi: 0.5022 | LR: 8.86e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0011 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3101 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.3718 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 3774.1 | uss_MB: 2989.4 | child_rss_MB: 4006.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:36:23,083] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:03,294] INFO : Step 11700/50000 rank0 | Train Loss: 9.9315 | Train Loss Unweighted: 3.1608 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.8222 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1434 | Regression_eta: 0.0143 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9602 | Classification_binary: 2.8222 | Regression_cos_phi: 0.7503 | Regression_energy: 1.9834 | Regression_eta: 0.7006 | Regression_pt: 2.0265 | Regression_sin_phi: 0.6883 | LR: 8.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.2524 | max_optimizer_step: 0.1138 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3772.1 | uss_MB: 2987.8 | child_rss_MB: 4003.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:37:03,509] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:37:43,387] INFO : Step 11800/50000 rank0 | Train Loss: 10.3230 | Train Loss Unweighted: 3.1269 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.7441 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1667 | Regression_eta: 0.0140 | Regression_pt: 0.1600 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8870 | Classification_binary: 2.7441 | Regression_cos_phi: 0.6702 | Regression_energy: 2.3051 | Regression_eta: 0.6875 | Regression_pt: 2.4071 | Regression_sin_phi: 0.6221 | LR: 8.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5684s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0032 | max_backward: 0.3090 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3773.1 | uss_MB: 2989.1 | child_rss_MB: 4003.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:37:43,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:38:23,511] INFO : Step 11900/50000 rank0 | Train Loss: 8.7285 | Train Loss Unweighted: 2.9571 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.6491 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1313 | Regression_eta: 0.0070 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9413 | Classification_binary: 2.6491 | Regression_cos_phi: 0.5704 | Regression_energy: 1.8162 | Regression_eta: 0.3421 | Regression_pt: 1.9711 | Regression_sin_phi: 0.4384 | LR: 8.80e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3092 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3773.1 | uss_MB: 2989.2 | child_rss_MB: 4002.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:38:23,726] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:03,153] INFO : Step 12000/50000 rank0 | Train Loss: 8.8581 | Train Loss Unweighted: 2.9674 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6506 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1360 | Regression_eta: 0.0091 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9102 | Classification_binary: 2.6506 | Regression_cos_phi: 0.5353 | Regression_energy: 1.8807 | Regression_eta: 0.4448 | Regression_pt: 2.0283 | Regression_sin_phi: 0.4082 | LR: 8.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.3994s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1148 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1135 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3773.1 | uss_MB: 2989.2 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:39:03,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:39:03,387] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-12000.pth +rank0 [2026-10-28 12:39:03,564] INFO : Running validation on rank0 +rank0 [2026-10-28 12:39:03,565] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:39:03,565] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:39:04,853] INFO : saved batch0_epoch12000.parquet +rank0 [2026-10-28 12:39:04,998] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:39:05,126] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:39:05,200] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:39:05,268] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:39:05,335] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:39:05,404] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:39:05,472] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:39:05,584] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:39:05,713] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:39:05,844] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:39:05,914] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:39:05,982] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:39:06,049] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:39:06,117] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:39:06,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:39:06,295] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:39:06,302] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:39:06,305] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:39:06,308] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:39:06,309] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:39:06,312] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:39:06,315] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:39:06,315] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:39:06,318] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:39:06,319] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:39:06,322] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:39:06,324] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:39:06,327] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:39:06,329] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:39:06,331] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:39:06,332] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:39:13,676] INFO : VALIDATION | Step=12000/50000 | Train Loss=8.8581 | Valid Loss=9.3530 | Train Loss Unweighted=2.9674 | Valid Loss Unweighted=3.0414 | Stale=0 | ETA=274.5m +rank0 [2026-10-28 12:39:13,685] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:13,769] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:39:13,773] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:13,928] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:15,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:16,586] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:17,300] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:18,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:18,104] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:39:18,107] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:18,165] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:39:18,168] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:18,255] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:20,100] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:20,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:21,406] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:22,052] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:22,168] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:22,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:39:22,242] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:39:22,245] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:39:22,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:39:24,122] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:39:24,795] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:39:25,477] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:39:26,147] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:39:26,245] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:39:26,253] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:26,502] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:39:26,945] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:28,812] INFO : Plotted jets +rank0 [2026-10-28 12:39:29,897] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:29,929] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:30,153] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:39:30,526] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:31,895] INFO : Plotted jets +rank0 [2026-10-28 12:39:32,950] INFO : Plotted jet ratio +rank0 [2026-10-28 12:39:32,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 12:39:33,184] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_12000/clic_edm_qq_pf +rank0 [2026-10-28 12:39:33,598] INFO : Plotted number of elements +rank0 [2026-10-28 12:39:34,973] INFO : Plotted jets +rank0 [2026-10-28 12:39:36,060] INFO : Plotted jet ratio +rank0 [2026-10-28 12:40:16,493] INFO : Step 12100/50000 rank0 | Train Loss: 9.3868 | Train Loss Unweighted: 3.0250 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6867 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1458 | Regression_eta: 0.0103 | Regression_pt: 0.1417 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9622 | Classification_binary: 2.6867 | Regression_cos_phi: 0.5895 | Regression_energy: 2.0165 | Regression_eta: 0.5063 | Regression_pt: 2.1322 | Regression_sin_phi: 0.4934 | LR: 8.76e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0505 | max_backward: 0.2029 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3695 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3845.6 | uss_MB: 3062.1 | child_rss_MB: 4001.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:40:16,708] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:40:56,180] INFO : Step 12200/50000 rank0 | Train Loss: 9.1465 | Train Loss Unweighted: 3.0782 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.7559 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1369 | Regression_eta: 0.0100 | Regression_pt: 0.1362 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9714 | Classification_binary: 2.7559 | Regression_cos_phi: 0.5194 | Regression_energy: 1.8938 | Regression_eta: 0.4907 | Regression_pt: 2.0492 | Regression_sin_phi: 0.4662 | LR: 8.74e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5143s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0853 | max_forward: 0.0780 | max_loss: 0.0032 | max_backward: 0.2031 | max_optimizer_step: 0.1793 | max_tensorboard_logging: 0.2776 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3846.6 | uss_MB: 3063.6 | child_rss_MB: 4006.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:40:56,405] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:41:35,956] INFO : Step 12300/50000 rank0 | Train Loss: 9.4439 | Train Loss Unweighted: 3.0429 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6909 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1536 | Regression_eta: 0.0087 | Regression_pt: 0.1518 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9124 | Classification_binary: 2.6909 | Regression_cos_phi: 0.5679 | Regression_energy: 2.1235 | Regression_eta: 0.4260 | Regression_pt: 2.2830 | Regression_sin_phi: 0.4402 | LR: 8.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5597s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.1039 | max_backward: 0.2034 | max_optimizer_step: 0.1170 | max_tensorboard_logging: 0.3628 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 3846.6 | uss_MB: 3063.6 | child_rss_MB: 4001.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:41:36,170] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 1% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:42:16,247] INFO : Step 12400/50000 rank0 | Train Loss: 9.4438 | Train Loss Unweighted: 3.0491 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7214 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1356 | Regression_eta: 0.0168 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9023 | Classification_binary: 2.7214 | Regression_cos_phi: 0.5846 | Regression_energy: 1.8756 | Regression_eta: 0.8249 | Regression_pt: 2.0552 | Regression_sin_phi: 0.4798 | LR: 8.70e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6177s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1830 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1171 | max_tensorboard_logging: 0.3626 | max_tensorboard_flush: 0.0262 | Memory: rss_MB: 3846.6 | uss_MB: 3063.7 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:42:16,465] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:42:55,550] INFO : Step 12500/50000 rank0 | Train Loss: 10.0017 | Train Loss Unweighted: 3.0837 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.7101 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1619 | Regression_eta: 0.0110 | Regression_pt: 0.1591 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9616 | Classification_binary: 2.7101 | Regression_cos_phi: 0.6415 | Regression_energy: 2.2383 | Regression_eta: 0.5415 | Regression_pt: 2.3943 | Regression_sin_phi: 0.5143 | LR: 8.68e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4218s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1333 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2742 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3846.6 | uss_MB: 3063.8 | child_rss_MB: 4003.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 12:42:55,766] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:43:35,908] INFO : Step 12600/50000 rank0 | Train Loss: 9.4728 | Train Loss Unweighted: 2.9968 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.6558 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1465 | Regression_eta: 0.0133 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8776 | Classification_binary: 2.6558 | Regression_cos_phi: 0.5718 | Regression_energy: 2.0255 | Regression_eta: 0.6545 | Regression_pt: 2.1382 | Regression_sin_phi: 0.5494 | LR: 8.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4566s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0377 | max_forward: 0.0774 | max_loss: 0.0031 | max_backward: 0.2028 | max_optimizer_step: 0.1648 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3846.6 | uss_MB: 3063.8 | child_rss_MB: 4006.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:43:36,131] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:15,980] INFO : Step 12700/50000 rank0 | Train Loss: 9.3671 | Train Loss Unweighted: 2.9971 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6666 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1421 | Regression_eta: 0.0169 | Regression_pt: 0.1345 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8436 | Classification_binary: 2.6666 | Regression_cos_phi: 0.5186 | Regression_energy: 1.9655 | Regression_eta: 0.8292 | Regression_pt: 2.0238 | Regression_sin_phi: 0.5198 | LR: 8.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4347s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1324 | max_loss: 0.0032 | max_backward: 0.2032 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.2787 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3845.5 | uss_MB: 3062.1 | child_rss_MB: 4004.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:44:16,194] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 41.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:44:56,030] INFO : Step 12800/50000 rank0 | Train Loss: 9.0291 | Train Loss Unweighted: 3.0914 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7666 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1407 | Regression_eta: 0.0083 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9605 | Classification_binary: 2.7666 | Regression_cos_phi: 0.4561 | Regression_energy: 1.9461 | Regression_eta: 0.4075 | Regression_pt: 2.0920 | Regression_sin_phi: 0.4003 | LR: 8.62e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6118s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0771 | max_loss: 0.0780 | max_backward: 0.2027 | max_optimizer_step: 0.1144 | max_tensorboard_logging: 0.4290 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 3847.5 | uss_MB: 3063.6 | child_rss_MB: 4004.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:44:56,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:45:35,981] INFO : Step 12900/50000 rank0 | Train Loss: 9.2413 | Train Loss Unweighted: 3.0749 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7472 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1396 | Regression_eta: 0.0093 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9096 | Classification_binary: 2.7472 | Regression_cos_phi: 0.5819 | Regression_energy: 1.9303 | Regression_eta: 0.4566 | Regression_pt: 2.0954 | Regression_sin_phi: 0.5204 | LR: 8.60e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4976s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0668 | max_backward: 0.2032 | max_optimizer_step: 0.1153 | max_tensorboard_logging: 0.3561 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3847.5 | uss_MB: 3063.7 | child_rss_MB: 4006.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:45:36,197] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:16,187] INFO : Step 13000/50000 rank0 | Train Loss: 9.9072 | Train Loss Unweighted: 3.1570 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.7955 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1558 | Regression_eta: 0.0112 | Regression_pt: 0.1530 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9497 | Classification_binary: 2.7955 | Regression_cos_phi: 0.5807 | Regression_energy: 2.1546 | Regression_eta: 0.5524 | Regression_pt: 2.3024 | Regression_sin_phi: 0.5718 | LR: 8.58e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4636s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1819 | max_loss: 0.0033 | max_backward: 0.2036 | max_optimizer_step: 0.1149 | max_tensorboard_logging: 0.2756 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 3847.5 | uss_MB: 3063.7 | child_rss_MB: 4003.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:46:16,398] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:46:16,434] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-13000.pth +rank0 [2026-10-28 12:46:16,606] INFO : Running validation on rank0 +rank0 [2026-10-28 12:46:16,607] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:46:16,607] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:46:17,889] INFO : saved batch0_epoch13000.parquet +rank0 [2026-10-28 12:46:18,034] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:46:18,163] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:46:18,236] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:46:18,305] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:46:18,372] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:46:18,441] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:46:18,510] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:46:18,620] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:46:18,749] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:46:18,879] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:46:18,949] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:46:19,017] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:46:19,084] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:46:19,153] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:46:19,221] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:46:19,328] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:46:19,335] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:46:19,338] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:46:19,341] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:46:19,342] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:46:19,344] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:46:19,347] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:46:19,348] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:46:19,349] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:46:19,351] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:46:19,354] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:46:19,355] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:46:19,357] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:46:19,360] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:46:19,361] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:46:19,363] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:46:26,645] INFO : VALIDATION | Step=13000/50000 | Train Loss=9.9072 | Valid Loss=9.3069 | Train Loss Unweighted=3.1570 | Valid Loss Unweighted=3.0420 | Stale=0 | ETA=267.2m +rank0 [2026-10-28 12:46:26,657] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:26,720] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:46:26,723] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:26,840] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:28,735] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:29,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:30,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:30,839] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:30,941] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:30,945] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:31,020] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:46:31,022] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:31,104] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:32,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:33,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:34,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:34,957] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:35,075] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:35,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:46:35,157] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:46:35,160] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:46:35,235] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:46:37,025] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:46:37,714] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:46:38,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:46:39,073] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:46:39,175] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:46:39,183] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:39,431] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:46:39,876] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:41,790] INFO : Plotted jets +rank0 [2026-10-28 12:46:42,882] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:42,916] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:43,138] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:46:43,509] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:44,876] INFO : Plotted jets +rank0 [2026-10-28 12:46:45,983] INFO : Plotted jet ratio +rank0 [2026-10-28 12:46:46,015] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 12:46:46,217] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_13000/clic_edm_qq_pf +rank0 [2026-10-28 12:46:46,628] INFO : Plotted number of elements +rank0 [2026-10-28 12:46:48,007] INFO : Plotted jets +rank0 [2026-10-28 12:46:49,091] INFO : Plotted jet ratio +rank0 [2026-10-28 12:47:29,303] INFO : Step 13100/50000 rank0 | Train Loss: 9.3780 | Train Loss Unweighted: 2.9958 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6629 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1401 | Regression_eta: 0.0099 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9140 | Classification_binary: 2.6629 | Regression_cos_phi: 0.6304 | Regression_energy: 1.9371 | Regression_eta: 0.4864 | Regression_pt: 2.1141 | Regression_sin_phi: 0.6330 | LR: 8.56e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5847s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.1040 | max_backward: 0.2033 | max_optimizer_step: 0.1204 | max_tensorboard_logging: 0.3471 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4069.6 | uss_MB: 3287.5 | child_rss_MB: 4002.3 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:47:29,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:09,301] INFO : Step 13200/50000 rank0 | Train Loss: 8.7494 | Train Loss Unweighted: 2.9192 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.6135 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1308 | Regression_eta: 0.0096 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8338 | Classification_binary: 2.6135 | Regression_cos_phi: 0.5659 | Regression_energy: 1.8086 | Regression_eta: 0.4725 | Regression_pt: 1.9170 | Regression_sin_phi: 0.5381 | LR: 8.54e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2562 | max_optimizer_step: 0.1196 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0127 | Memory: rss_MB: 3921.5 | uss_MB: 3138.0 | child_rss_MB: 4007.3 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:48:09,513] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:48:49,083] INFO : Step 13300/50000 rank0 | Train Loss: 9.5944 | Train Loss Unweighted: 3.0840 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.7284 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1536 | Regression_eta: 0.0095 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9386 | Classification_binary: 2.7284 | Regression_cos_phi: 0.5499 | Regression_energy: 2.1235 | Regression_eta: 0.4667 | Regression_pt: 2.3093 | Regression_sin_phi: 0.4780 | LR: 8.52e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5070s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1848 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.2790 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 3921.5 | uss_MB: 3137.6 | child_rss_MB: 4003.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:48:49,299] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:49:29,538] INFO : Step 13400/50000 rank0 | Train Loss: 9.5045 | Train Loss Unweighted: 3.0889 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7552 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1416 | Regression_eta: 0.0108 | Regression_pt: 0.1391 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9590 | Classification_binary: 2.7552 | Regression_cos_phi: 0.7123 | Regression_energy: 1.9578 | Regression_eta: 0.5299 | Regression_pt: 2.0921 | Regression_sin_phi: 0.4981 | LR: 8.50e-04 | DataLoad Time: 0.0034s | Model Forward Time: 1.4848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0032 | max_backward: 0.3100 | max_optimizer_step: 0.1209 | max_tensorboard_logging: 0.2708 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3921.5 | uss_MB: 3137.6 | child_rss_MB: 4034.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:49:29,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:09,108] INFO : Step 13500/50000 rank0 | Train Loss: 9.4437 | Train Loss Unweighted: 3.0760 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.7339 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1472 | Regression_eta: 0.0098 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9085 | Classification_binary: 2.7339 | Regression_cos_phi: 0.5700 | Regression_energy: 2.0350 | Regression_eta: 0.4823 | Regression_pt: 2.1930 | Regression_sin_phi: 0.5210 | LR: 8.48e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8302s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1055 | max_forward: 0.1046 | max_loss: 0.0033 | max_backward: 0.3950 | max_optimizer_step: 0.1200 | max_tensorboard_logging: 0.3957 | max_tensorboard_flush: 0.0035 | Memory: rss_MB: 3921.5 | uss_MB: 3140.2 | child_rss_MB: 4006.9 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 12:50:09,322] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:50:48,790] INFO : Step 13600/50000 rank0 | Train Loss: 9.8253 | Train Loss Unweighted: 3.0410 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.6743 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1588 | Regression_eta: 0.0104 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8788 | Classification_binary: 2.6743 | Regression_cos_phi: 0.6764 | Regression_energy: 2.1954 | Regression_eta: 0.5108 | Regression_pt: 2.3663 | Regression_sin_phi: 0.5231 | LR: 8.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5144s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3921.5 | uss_MB: 3138.1 | child_rss_MB: 4005.6 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:50:49,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:51:29,480] INFO : Step 13700/50000 rank0 | Train Loss: 9.3738 | Train Loss Unweighted: 3.0819 | Train Loss Components: Classification: 0.0244 | Classification_binary: 2.7559 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1367 | Regression_eta: 0.0116 | Regression_pt: 0.1358 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 1.0230 | Classification_binary: 2.7559 | Regression_cos_phi: 0.6355 | Regression_energy: 1.8910 | Regression_eta: 0.5697 | Regression_pt: 2.0428 | Regression_sin_phi: 0.4559 | LR: 8.43e-04 | DataLoad Time: 0.0032s | Model Forward Time: 1.7337s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1051 | max_loss: 0.0033 | max_backward: 0.3926 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3892 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3921.5 | uss_MB: 3140.2 | child_rss_MB: 4009.0 | allocated_MB: 295.1 | reserved_MB: 3784.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 12:51:29,696] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:10,375] INFO : Step 13800/50000 rank0 | Train Loss: 9.2593 | Train Loss Unweighted: 2.9632 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.6360 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1385 | Regression_eta: 0.0105 | Regression_pt: 0.1364 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9374 | Classification_binary: 2.6360 | Regression_cos_phi: 0.6473 | Regression_energy: 1.9157 | Regression_eta: 0.5141 | Regression_pt: 2.0523 | Regression_sin_phi: 0.5565 | LR: 8.41e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0032 | max_backward: 0.3080 | max_optimizer_step: 0.1186 | max_tensorboard_logging: 0.3881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3920.3 | uss_MB: 3137.4 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:52:10,590] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:52:50,474] INFO : Step 13900/50000 rank0 | Train Loss: 9.1070 | Train Loss Unweighted: 3.0335 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.7213 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1326 | Regression_eta: 0.0111 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9435 | Classification_binary: 2.7213 | Regression_cos_phi: 0.5938 | Regression_energy: 1.8330 | Regression_eta: 0.5450 | Regression_pt: 1.9175 | Regression_sin_phi: 0.5529 | LR: 8.39e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5197s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0033 | max_backward: 0.2230 | max_optimizer_step: 0.1166 | max_tensorboard_logging: 0.3928 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 3920.3 | uss_MB: 3137.6 | child_rss_MB: 4012.1 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:52:50,688] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:30,217] INFO : Step 14000/50000 rank0 | Train Loss: 9.1316 | Train Loss Unweighted: 2.9412 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.6177 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1398 | Regression_eta: 0.0131 | Regression_pt: 0.1334 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7860 | Classification_binary: 2.6177 | Regression_cos_phi: 0.6367 | Regression_energy: 1.9336 | Regression_eta: 0.6418 | Regression_pt: 2.0073 | Regression_sin_phi: 0.5085 | LR: 8.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5050s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0266 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2031 | max_optimizer_step: 0.1801 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 3920.3 | uss_MB: 3137.7 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3784.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:53:30,440] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:53:30,459] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-14000.pth +rank0 [2026-10-28 12:53:30,632] INFO : Running validation on rank0 +rank0 [2026-10-28 12:53:30,632] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 12:53:30,632] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 12:53:31,964] INFO : saved batch0_epoch14000.parquet +rank0 [2026-10-28 12:53:32,107] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 12:53:32,237] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 12:53:32,309] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 12:53:32,377] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 12:53:32,445] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 12:53:32,515] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 12:53:32,583] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 12:53:32,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 12:53:32,825] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 12:53:32,955] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 12:53:33,025] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 12:53:33,094] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 12:53:33,162] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 12:53:33,231] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 12:53:33,300] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 12:53:33,407] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 12:53:33,414] INFO : plotted pt_target histogram +rank0 [2026-10-28 12:53:33,417] INFO : plotted pt_pred histogram +rank0 [2026-10-28 12:53:33,420] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 12:53:33,421] INFO : plotted eta_target histogram +rank0 [2026-10-28 12:53:33,424] INFO : plotted eta_pred histogram +rank0 [2026-10-28 12:53:33,427] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 12:53:33,430] INFO : plotted sphi_target histogram +rank0 [2026-10-28 12:53:33,431] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 12:53:33,434] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 12:53:33,436] INFO : plotted cphi_target histogram +rank0 [2026-10-28 12:53:33,438] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 12:53:33,440] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 12:53:33,442] INFO : plotted energy_target histogram +rank0 [2026-10-28 12:53:33,445] INFO : plotted energy_pred histogram +rank0 [2026-10-28 12:53:33,446] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 12:53:41,021] INFO : VALIDATION | Step=14000/50000 | Train Loss=9.1316 | Valid Loss=9.2182 | Train Loss Unweighted=2.9412 | Valid Loss Unweighted=2.9859 | Stale=0 | ETA=260.0m +rank0 [2026-10-28 12:53:41,032] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:41,094] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 12:53:41,097] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:41,252] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:43,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:43,903] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:44,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:45,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:45,426] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:53:45,429] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:45,485] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 12:53:45,488] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:45,577] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:47,441] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:48,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:48,783] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:49,465] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:49,587] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 12:53:49,591] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 12:53:49,645] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 12:53:49,648] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 12:53:49,753] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 12:53:51,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 12:53:52,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 12:53:52,899] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 12:53:53,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 12:53:53,677] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 12:53:53,685] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:53,937] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ttbar_pf +rank0 [2026-10-28 12:53:54,383] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:55,803] INFO : Plotted jets +rank0 [2026-10-28 12:53:56,881] INFO : Plotted jet ratio +rank0 [2026-10-28 12:53:56,914] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:57,135] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 12:53:57,507] INFO : Plotted number of elements +rank0 [2026-10-28 12:53:59,377] INFO : Plotted jets +rank0 [2026-10-28 12:54:00,492] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:00,524] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:00,724] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_14000/clic_edm_qq_pf +rank0 [2026-10-28 12:54:01,136] INFO : Plotted number of elements +rank0 [2026-10-28 12:54:02,512] INFO : Plotted jets +rank0 [2026-10-28 12:54:03,611] INFO : Plotted jet ratio +rank0 [2026-10-28 12:54:43,612] INFO : Step 14100/50000 rank0 | Train Loss: 9.3844 | Train Loss Unweighted: 2.9438 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6077 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1424 | Regression_eta: 0.0108 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9504 | Classification_binary: 2.6077 | Regression_cos_phi: 0.5913 | Regression_energy: 1.9694 | Regression_eta: 0.5310 | Regression_pt: 2.1121 | Regression_sin_phi: 0.6227 | LR: 8.35e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5405s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0729 | max_forward: 0.0779 | max_loss: 0.0033 | max_backward: 0.2032 | max_optimizer_step: 0.1568 | max_tensorboard_logging: 0.2769 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4123.5 | uss_MB: 3342.0 | child_rss_MB: 4008.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:54:43,838] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:55:23,517] INFO : Step 14200/50000 rank0 | Train Loss: 9.4260 | Train Loss Unweighted: 3.0011 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.6655 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1452 | Regression_eta: 0.0128 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8619 | Classification_binary: 2.6655 | Regression_cos_phi: 0.5978 | Regression_energy: 2.0083 | Regression_eta: 0.6269 | Regression_pt: 2.0711 | Regression_sin_phi: 0.5945 | LR: 8.32e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5354s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0428 | max_backward: 0.2035 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3683 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 3997.6 | uss_MB: 3215.4 | child_rss_MB: 4007.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:55:23,729] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:03,651] INFO : Step 14300/50000 rank0 | Train Loss: 9.5671 | Train Loss Unweighted: 2.9844 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6255 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1556 | Regression_eta: 0.0109 | Regression_pt: 0.1536 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9523 | Classification_binary: 2.6255 | Regression_cos_phi: 0.5396 | Regression_energy: 2.1518 | Regression_eta: 0.5369 | Regression_pt: 2.3111 | Regression_sin_phi: 0.4498 | LR: 8.30e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5144s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0461 | max_backward: 0.2031 | max_optimizer_step: 0.1181 | max_tensorboard_logging: 0.3630 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 3997.6 | uss_MB: 3215.6 | child_rss_MB: 4015.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:56:03,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:56:43,797] INFO : Step 14400/50000 rank0 | Train Loss: 9.2978 | Train Loss Unweighted: 2.9826 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6399 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1488 | Regression_eta: 0.0103 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9530 | Classification_binary: 2.6399 | Regression_cos_phi: 0.5056 | Regression_energy: 2.0573 | Regression_eta: 0.5061 | Regression_pt: 2.1885 | Regression_sin_phi: 0.4475 | LR: 8.28e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.5807s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.1006 | max_backward: 0.2032 | max_optimizer_step: 0.1191 | max_tensorboard_logging: 0.3692 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 3997.6 | uss_MB: 3215.6 | child_rss_MB: 4013.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:56:44,009] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:57:23,900] INFO : Step 14500/50000 rank0 | Train Loss: 9.0609 | Train Loss Unweighted: 2.9192 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1386 | Regression_eta: 0.0091 | Regression_pt: 0.1419 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9417 | Classification_binary: 2.5905 | Regression_cos_phi: 0.5683 | Regression_energy: 1.9166 | Regression_eta: 0.4452 | Regression_pt: 2.1346 | Regression_sin_phi: 0.4640 | LR: 8.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4953s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0330 | max_backward: 0.2033 | max_optimizer_step: 0.1194 | max_tensorboard_logging: 0.3560 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3215.7 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 12:57:24,113] INFO : + +=========================================== ROCm System Management Interface =========================================== +===================================================== Concise Info ===================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +======================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 1650Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +======================================================================================================================== +================================================= End of ROCm SMI Log ================================================== + +rank0 [2026-10-28 12:58:04,280] INFO : Step 14600/50000 rank0 | Train Loss: 9.3270 | Train Loss Unweighted: 3.0363 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7036 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1416 | Regression_eta: 0.0097 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9244 | Classification_binary: 2.7036 | Regression_cos_phi: 0.5812 | Regression_energy: 1.9579 | Regression_eta: 0.4772 | Regression_pt: 2.1174 | Regression_sin_phi: 0.5653 | LR: 8.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5757s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1575 | max_loss: 0.0033 | max_backward: 0.2029 | max_optimizer_step: 0.2270 | max_tensorboard_logging: 0.2735 | max_tensorboard_flush: 0.0089 | Memory: rss_MB: 3997.6 | uss_MB: 3215.8 | child_rss_MB: 4009.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:58:04,505] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:58:44,881] INFO : Step 14700/50000 rank0 | Train Loss: 9.1428 | Train Loss Unweighted: 2.9849 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6524 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1435 | Regression_eta: 0.0111 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8871 | Classification_binary: 2.6524 | Regression_cos_phi: 0.5322 | Regression_energy: 1.9849 | Regression_eta: 0.5437 | Regression_pt: 2.1297 | Regression_sin_phi: 0.4129 | LR: 8.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4863s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.1202 | max_tensorboard_logging: 0.2701 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 3997.6 | uss_MB: 3215.8 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 12:58:45,093] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 12:59:25,000] INFO : Step 14800/50000 rank0 | Train Loss: 9.1982 | Train Loss Unweighted: 3.0344 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.7157 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1341 | Regression_eta: 0.0110 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9362 | Classification_binary: 2.7157 | Regression_cos_phi: 0.6078 | Regression_energy: 1.8550 | Regression_eta: 0.5383 | Regression_pt: 1.9922 | Regression_sin_phi: 0.5530 | LR: 8.19e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4734s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0015 | max_forward: 0.0778 | max_loss: 0.0242 | max_backward: 0.2031 | max_optimizer_step: 0.1185 | max_tensorboard_logging: 0.3568 | max_tensorboard_flush: 0.0091 | Memory: rss_MB: 3997.7 | uss_MB: 3215.6 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 12:59:25,213] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:05,106] INFO : Step 14900/50000 rank0 | Train Loss: 9.6396 | Train Loss Unweighted: 3.0810 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.7351 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1478 | Regression_eta: 0.0104 | Regression_pt: 0.1458 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9001 | Classification_binary: 2.7351 | Regression_cos_phi: 0.6337 | Regression_energy: 2.0438 | Regression_eta: 0.5089 | Regression_pt: 2.1932 | Regression_sin_phi: 0.6247 | LR: 8.17e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.4651s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0198 | max_backward: 0.2032 | max_optimizer_step: 0.1179 | max_tensorboard_logging: 0.3512 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 3997.7 | uss_MB: 3215.6 | child_rss_MB: 4010.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:00:05,320] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:44,760] INFO : Step 15000/50000 rank0 | Train Loss: 8.1627 | Train Loss Unweighted: 2.7987 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5168 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1191 | Regression_eta: 0.0100 | Regression_pt: 0.1179 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8974 | Classification_binary: 2.5168 | Regression_cos_phi: 0.4549 | Regression_energy: 1.6473 | Regression_eta: 0.4930 | Regression_pt: 1.7738 | Regression_sin_phi: 0.3796 | LR: 8.15e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5088s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0946 | max_forward: 0.0779 | max_loss: 0.0032 | max_backward: 0.2033 | max_optimizer_step: 0.1668 | max_tensorboard_logging: 0.2747 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 3997.7 | uss_MB: 3215.8 | child_rss_MB: 4009.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:00:44,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:00:45,003] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-15000.pth +rank0 [2026-10-28 13:00:45,175] INFO : Running validation on rank0 +rank0 [2026-10-28 13:00:45,175] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:00:45,175] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:00:46,465] INFO : saved batch0_epoch15000.parquet +rank0 [2026-10-28 13:00:46,608] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:00:46,736] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:00:46,809] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:00:46,877] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:00:46,945] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:00:47,016] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:00:47,084] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:00:47,196] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:00:47,325] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:00:47,455] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:00:47,525] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:00:47,594] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:00:47,663] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:00:47,731] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:00:47,800] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:00:47,911] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:00:47,918] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:00:47,921] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:00:47,924] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:00:47,927] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:00:47,927] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:00:47,930] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:00:47,931] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:00:47,934] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:00:47,935] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:00:47,938] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:00:47,938] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:00:47,941] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:00:47,944] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:00:47,947] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:00:47,950] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:00:55,529] INFO : VALIDATION | Step=15000/50000 | Train Loss=8.1627 | Valid Loss=9.1146 | Train Loss Unweighted=2.7987 | Valid Loss Unweighted=2.9645 | Stale=0 | ETA=252.9m +rank0 [2026-10-28 13:00:55,540] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:55,602] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:00:55,607] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:00:55,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:00:57,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:00:58,214] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:00:58,848] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:00:59,464] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:00:59,572] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:00:59,575] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:00:59,636] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:00:59,639] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:00:59,726] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:01,494] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:02,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:02,849] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:03,526] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:03,640] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:01:03,643] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:01:03,696] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:01:03,699] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:01:03,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:01:05,595] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:01:06,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:01:06,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:01:07,627] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:01:07,756] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:01:07,764] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:08,015] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:01:08,461] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:10,344] INFO : Plotted jets +rank0 [2026-10-28 13:01:11,437] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:11,470] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:11,695] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:01:12,066] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:13,439] INFO : Plotted jets +rank0 [2026-10-28 13:01:14,551] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:14,585] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:14,787] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_15000/clic_edm_qq_pf +rank0 [2026-10-28 13:01:15,199] INFO : Plotted number of elements +rank0 [2026-10-28 13:01:16,581] INFO : Plotted jets +rank0 [2026-10-28 13:01:17,669] INFO : Plotted jet ratio +rank0 [2026-10-28 13:01:57,890] INFO : Step 15100/50000 rank0 | Train Loss: 8.8114 | Train Loss Unweighted: 2.9253 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.6249 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1246 | Regression_eta: 0.0088 | Regression_pt: 0.1248 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9486 | Classification_binary: 2.6249 | Regression_cos_phi: 0.6174 | Regression_energy: 1.7226 | Regression_eta: 0.4330 | Regression_pt: 1.8771 | Regression_sin_phi: 0.5877 | LR: 8.12e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.6338s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1248 | max_tensorboard_logging: 0.3743 | max_tensorboard_flush: 0.0054 | Memory: rss_MB: 4210.2 | uss_MB: 3430.0 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:01:58,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:02:37,390] INFO : Step 15200/50000 rank0 | Train Loss: 8.9394 | Train Loss Unweighted: 2.9543 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.6455 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1305 | Regression_eta: 0.0099 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9633 | Classification_binary: 2.6455 | Regression_cos_phi: 0.6335 | Regression_energy: 1.8048 | Regression_eta: 0.4843 | Regression_pt: 1.9167 | Regression_sin_phi: 0.4913 | LR: 8.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.3096 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.3654 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4068.4 | uss_MB: 3287.8 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:02:37,603] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:16,876] INFO : Step 15300/50000 rank0 | Train Loss: 9.9386 | Train Loss Unweighted: 3.0629 | Train Loss Components: Classification: 0.0227 | Classification_binary: 2.6988 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1560 | Regression_eta: 0.0116 | Regression_pt: 0.1535 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9494 | Classification_binary: 2.6988 | Regression_cos_phi: 0.6470 | Regression_energy: 2.1571 | Regression_eta: 0.5702 | Regression_pt: 2.3101 | Regression_sin_phi: 0.6060 | LR: 8.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.2948 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0088 | Memory: rss_MB: 4068.4 | uss_MB: 3288.0 | child_rss_MB: 4004.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:03:17,091] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:03:57,155] INFO : Step 15400/50000 rank0 | Train Loss: 9.1750 | Train Loss Unweighted: 2.9511 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6208 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1409 | Regression_eta: 0.0104 | Regression_pt: 0.1396 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9112 | Classification_binary: 2.6208 | Regression_cos_phi: 0.5142 | Regression_energy: 1.9485 | Regression_eta: 0.5128 | Regression_pt: 2.0996 | Regression_sin_phi: 0.5679 | LR: 8.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1045 | max_loss: 0.0034 | max_backward: 0.3207 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.3485 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4069.4 | uss_MB: 3290.5 | child_rss_MB: 4003.8 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 92.9 +rank0 [2026-10-28 13:03:57,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:04:36,848] INFO : Step 15500/50000 rank0 | Train Loss: 8.7099 | Train Loss Unweighted: 2.8486 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.5340 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1358 | Regression_eta: 0.0094 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9659 | Classification_binary: 2.5340 | Regression_cos_phi: 0.5230 | Regression_energy: 1.8780 | Regression_eta: 0.4626 | Regression_pt: 1.9894 | Regression_sin_phi: 0.3570 | LR: 8.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4604s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0263 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.2026 | max_optimizer_step: 0.1663 | max_tensorboard_logging: 0.2757 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4069.4 | uss_MB: 3288.5 | child_rss_MB: 4010.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:04:37,073] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:16,814] INFO : Step 15600/50000 rank0 | Train Loss: 9.2667 | Train Loss Unweighted: 2.9987 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6570 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1489 | Regression_eta: 0.0090 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 2.6570 | Regression_cos_phi: 0.5468 | Regression_energy: 2.0593 | Regression_eta: 0.4407 | Regression_pt: 2.2013 | Regression_sin_phi: 0.5041 | LR: 8.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5040s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1831 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1210 | max_tensorboard_logging: 0.2760 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4069.4 | uss_MB: 3288.6 | child_rss_MB: 4006.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:05:17,030] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 42.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:05:56,261] INFO : Step 15700/50000 rank0 | Train Loss: 9.1317 | Train Loss Unweighted: 2.9941 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.6754 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1349 | Regression_eta: 0.0112 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9136 | Classification_binary: 2.6754 | Regression_cos_phi: 0.5757 | Regression_energy: 1.8653 | Regression_eta: 0.5525 | Regression_pt: 1.9898 | Regression_sin_phi: 0.5594 | LR: 7.98e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4231s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1091 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1188 | max_tensorboard_logging: 0.2745 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4069.4 | uss_MB: 3288.7 | child_rss_MB: 4004.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:05:56,479] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:06:36,516] INFO : Step 15800/50000 rank0 | Train Loss: 8.9628 | Train Loss Unweighted: 2.8917 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5645 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1402 | Regression_eta: 0.0077 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9176 | Classification_binary: 2.5645 | Regression_cos_phi: 0.5680 | Regression_energy: 1.9384 | Regression_eta: 0.3797 | Regression_pt: 2.1140 | Regression_sin_phi: 0.4807 | LR: 7.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4516s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0464 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1662 | max_tensorboard_logging: 0.2654 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4069.4 | uss_MB: 3287.7 | child_rss_MB: 4007.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:06:36,742] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:16,922] INFO : Step 15900/50000 rank0 | Train Loss: 9.5369 | Train Loss Unweighted: 3.0345 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6952 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1429 | Regression_eta: 0.0117 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9073 | Classification_binary: 2.6952 | Regression_cos_phi: 0.6641 | Regression_energy: 1.9763 | Regression_eta: 0.5766 | Regression_pt: 2.1530 | Regression_sin_phi: 0.5643 | LR: 7.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5706s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3077 | max_optimizer_step: 0.1195 | max_tensorboard_logging: 0.3666 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4069.4 | uss_MB: 3288.0 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:17,136] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:56,960] INFO : Step 16000/50000 rank0 | Train Loss: 9.2916 | Train Loss Unweighted: 2.9937 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.6671 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1377 | Regression_eta: 0.0116 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9163 | Classification_binary: 2.6671 | Regression_cos_phi: 0.6967 | Regression_energy: 1.9036 | Regression_eta: 0.5678 | Regression_pt: 2.0544 | Regression_sin_phi: 0.4857 | LR: 7.91e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5514s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0033 | max_backward: 0.3028 | max_optimizer_step: 0.1182 | max_tensorboard_logging: 0.3594 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4069.4 | uss_MB: 3288.3 | child_rss_MB: 4001.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:07:57,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:07:57,191] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-16000.pth +rank0 [2026-10-28 13:07:57,466] INFO : Running validation on rank0 +rank0 [2026-10-28 13:07:57,466] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:07:57,466] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:07:58,839] INFO : saved batch0_epoch16000.parquet +rank0 [2026-10-28 13:07:58,982] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:07:59,112] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:07:59,184] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:07:59,252] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:07:59,319] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:07:59,388] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:07:59,457] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:07:59,568] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:07:59,699] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:07:59,828] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:07:59,898] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:07:59,967] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:08:00,035] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:08:00,105] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:08:00,173] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:08:00,289] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:08:00,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:08:00,299] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:08:00,302] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:08:00,303] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:08:00,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:08:00,309] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:08:00,312] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:08:00,313] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:08:00,315] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:08:00,316] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:08:00,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:08:00,320] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:08:00,322] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:08:00,323] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:08:00,326] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:08:07,659] INFO : VALIDATION | Step=16000/50000 | Train Loss=9.2916 | Valid Loss=9.1660 | Train Loss Unweighted=2.9937 | Valid Loss Unweighted=2.9781 | Stale=1 | ETA=245.6m +rank0 [2026-10-28 13:08:07,668] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:07,766] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:08:07,770] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:07,930] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:09,887] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:10,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:11,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:12,194] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:12,311] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:08:12,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:12,371] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:08:12,374] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:12,468] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:14,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:14,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:15,452] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:16,104] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:16,221] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:16,224] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:08:16,284] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:08:16,286] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:08:16,375] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:08:18,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:08:18,833] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:08:19,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:08:20,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:08:20,309] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:08:20,317] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:20,567] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:08:21,015] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:22,959] INFO : Plotted jets +rank0 [2026-10-28 13:08:24,054] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:24,087] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:24,311] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:08:24,680] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:26,051] INFO : Plotted jets +rank0 [2026-10-28 13:08:27,162] INFO : Plotted jet ratio +rank0 [2026-10-28 13:08:27,196] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:27,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_16000/clic_edm_qq_pf +rank0 [2026-10-28 13:08:27,810] INFO : Plotted number of elements +rank0 [2026-10-28 13:08:29,195] INFO : Plotted jets +rank0 [2026-10-28 13:08:30,286] INFO : Plotted jet ratio +rank0 [2026-10-28 13:09:10,600] INFO : Step 16100/50000 rank0 | Train Loss: 8.9912 | Train Loss Unweighted: 2.9049 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5937 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1329 | Regression_eta: 0.0109 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8786 | Classification_binary: 2.5937 | Regression_cos_phi: 0.6501 | Regression_energy: 1.8372 | Regression_eta: 0.5335 | Regression_pt: 1.9007 | Regression_sin_phi: 0.5974 | LR: 7.89e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5332s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1233 | max_tensorboard_logging: 0.2709 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4296.1 | uss_MB: 3515.9 | child_rss_MB: 4005.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:09:10,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:09:50,248] INFO : Step 16200/50000 rank0 | Train Loss: 9.5920 | Train Loss Unweighted: 3.0162 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.6767 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1412 | Regression_eta: 0.0125 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9709 | Classification_binary: 2.6767 | Regression_cos_phi: 0.6829 | Regression_energy: 1.9523 | Regression_eta: 0.6153 | Regression_pt: 2.1489 | Regression_sin_phi: 0.5450 | LR: 7.86e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5402s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0697 | max_forward: 0.0773 | max_loss: 0.0032 | max_backward: 0.2027 | max_optimizer_step: 0.1588 | max_tensorboard_logging: 0.2845 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4142.3 | uss_MB: 3360.8 | child_rss_MB: 4002.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:09:50,475] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:10:30,140] INFO : Step 16300/50000 rank0 | Train Loss: 8.2695 | Train Loss Unweighted: 2.7899 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.5026 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1219 | Regression_eta: 0.0099 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8643 | Classification_binary: 2.5026 | Regression_cos_phi: 0.4970 | Regression_energy: 1.6850 | Regression_eta: 0.4877 | Regression_pt: 1.8037 | Regression_sin_phi: 0.4291 | LR: 7.84e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.1001 | max_backward: 0.2029 | max_optimizer_step: 0.1220 | max_tensorboard_logging: 0.3389 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4142.3 | uss_MB: 3361.1 | child_rss_MB: 4001.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:10:30,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:10,273] INFO : Step 16400/50000 rank0 | Train Loss: 9.1104 | Train Loss Unweighted: 2.9067 | Train Loss Components: Classification: 0.0237 | Classification_binary: 2.5844 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1364 | Regression_eta: 0.0127 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9913 | Classification_binary: 2.5844 | Regression_cos_phi: 0.5962 | Regression_energy: 1.8867 | Regression_eta: 0.6249 | Regression_pt: 2.0038 | Regression_sin_phi: 0.4231 | LR: 7.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5206s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0559 | max_backward: 0.2034 | max_optimizer_step: 0.1213 | max_tensorboard_logging: 0.3467 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4142.3 | uss_MB: 3361.1 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:11:10,488] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:11:50,405] INFO : Step 16500/50000 rank0 | Train Loss: 9.1897 | Train Loss Unweighted: 3.0006 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.6727 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1388 | Regression_eta: 0.0105 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8532 | Classification_binary: 2.6727 | Regression_cos_phi: 0.6446 | Regression_energy: 1.9200 | Regression_eta: 0.5141 | Regression_pt: 2.1089 | Regression_sin_phi: 0.4760 | LR: 7.79e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.4768s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1855 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1178 | max_tensorboard_logging: 0.2666 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4142.3 | uss_MB: 3361.2 | child_rss_MB: 4002.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:11:50,619] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:12:30,440] INFO : Step 16600/50000 rank0 | Train Loss: 8.6120 | Train Loss Unweighted: 2.8724 | Train Loss Components: Classification: 0.0224 | Classification_binary: 2.5689 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1279 | Regression_eta: 0.0073 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9398 | Classification_binary: 2.5689 | Regression_cos_phi: 0.5767 | Regression_energy: 1.7693 | Regression_eta: 0.3591 | Regression_pt: 1.9439 | Regression_sin_phi: 0.4543 | LR: 7.77e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5063s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2215 | max_optimizer_step: 0.1214 | max_tensorboard_logging: 0.3639 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4142.3 | uss_MB: 3361.2 | child_rss_MB: 4001.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:12:30,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:10,996] INFO : Step 16700/50000 rank0 | Train Loss: 8.8424 | Train Loss Unweighted: 2.8430 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5357 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1281 | Regression_eta: 0.0101 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9071 | Classification_binary: 2.5357 | Regression_cos_phi: 0.6199 | Regression_energy: 1.7715 | Regression_eta: 0.4979 | Regression_pt: 1.9230 | Regression_sin_phi: 0.5872 | LR: 7.74e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.5761s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1437 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2296 | max_tensorboard_logging: 0.2770 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4142.3 | uss_MB: 3361.3 | child_rss_MB: 4008.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:13:11,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:13:51,411] INFO : Step 16800/50000 rank0 | Train Loss: 9.4870 | Train Loss Unweighted: 2.9813 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.6361 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1477 | Regression_eta: 0.0104 | Regression_pt: 0.1460 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9234 | Classification_binary: 2.6361 | Regression_cos_phi: 0.6189 | Regression_energy: 2.0426 | Regression_eta: 0.5092 | Regression_pt: 2.1972 | Regression_sin_phi: 0.5596 | LR: 7.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4580s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1656 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1177 | max_tensorboard_logging: 0.2720 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 4142.5 | uss_MB: 3360.9 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:13:51,628] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:14:31,639] INFO : Step 16900/50000 rank0 | Train Loss: 9.1282 | Train Loss Unweighted: 2.9636 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.6367 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1397 | Regression_eta: 0.0096 | Regression_pt: 0.1385 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8731 | Classification_binary: 2.6367 | Regression_cos_phi: 0.6440 | Regression_energy: 1.9318 | Regression_eta: 0.4739 | Regression_pt: 2.0833 | Regression_sin_phi: 0.4854 | LR: 7.69e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5318s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0659 | max_backward: 0.2030 | max_optimizer_step: 0.1172 | max_tensorboard_logging: 0.3613 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4142.5 | uss_MB: 3361.0 | child_rss_MB: 4006.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:14:31,855] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:11,173] INFO : Step 17000/50000 rank0 | Train Loss: 9.5655 | Train Loss Unweighted: 2.9205 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5604 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1534 | Regression_eta: 0.0112 | Regression_pt: 0.1572 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.5604 | Regression_cos_phi: 0.6133 | Regression_energy: 2.1208 | Regression_eta: 0.5502 | Regression_pt: 2.3645 | Regression_sin_phi: 0.5049 | LR: 7.67e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.4073s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1000 | max_loss: 0.0033 | max_backward: 0.2033 | max_optimizer_step: 0.1193 | max_tensorboard_logging: 0.2715 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4142.5 | uss_MB: 3361.3 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:15:11,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:15:11,407] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-17000.pth +rank0 [2026-10-28 13:15:11,583] INFO : Running validation on rank0 +rank0 [2026-10-28 13:15:11,583] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:15:11,583] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:15:12,871] INFO : saved batch0_epoch17000.parquet +rank0 [2026-10-28 13:15:13,014] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:15:13,142] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:15:13,215] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:15:13,284] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:15:13,352] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:15:13,421] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:15:13,488] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:15:13,602] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:15:13,732] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:15:13,861] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:15:13,932] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:15:14,001] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:15:14,069] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:15:14,137] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:15:14,204] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:15:14,322] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:15:14,329] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:15:14,332] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:15:14,333] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:15:14,335] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:15:14,338] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:15:14,339] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:15:14,342] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:15:14,345] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:15:14,348] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:15:14,349] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:15:14,351] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:15:14,354] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:15:14,354] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:15:14,357] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:15:14,360] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:15:21,641] INFO : VALIDATION | Step=17000/50000 | Train Loss=9.5655 | Valid Loss=9.0179 | Train Loss Unweighted=2.9205 | Valid Loss Unweighted=2.9269 | Stale=0 | ETA=238.4m +rank0 [2026-10-28 13:15:21,652] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:21,732] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:15:21,736] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:21,852] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:23,775] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:24,506] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:25,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:25,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:25,981] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:25,984] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:26,049] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:15:26,052] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:26,134] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:27,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:28,647] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:29,341] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:30,024] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:30,144] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:30,147] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:15:30,201] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:15:30,204] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:15:30,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:15:32,099] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:15:32,770] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:15:33,455] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:15:34,130] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:15:34,252] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:15:34,259] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:34,511] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:15:34,957] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:36,375] INFO : Plotted jets +rank0 [2026-10-28 13:15:37,972] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:38,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:38,231] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:15:38,599] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:39,970] INFO : Plotted jets +rank0 [2026-10-28 13:15:41,079] INFO : Plotted jet ratio +rank0 [2026-10-28 13:15:41,111] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:41,318] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_17000/clic_edm_qq_pf +rank0 [2026-10-28 13:15:41,732] INFO : Plotted number of elements +rank0 [2026-10-28 13:15:43,119] INFO : Plotted jets +rank0 [2026-10-28 13:15:44,214] INFO : Plotted jet ratio +rank0 [2026-10-28 13:16:24,999] INFO : Step 17100/50000 rank0 | Train Loss: 8.8490 | Train Loss Unweighted: 3.0253 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.7086 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1377 | Regression_eta: 0.0088 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8794 | Classification_binary: 2.7086 | Regression_cos_phi: 0.4695 | Regression_energy: 1.9047 | Regression_eta: 0.4345 | Regression_pt: 2.0249 | Regression_sin_phi: 0.4275 | LR: 7.64e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6784s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1294 | max_loss: 0.0034 | max_backward: 0.3884 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.2828 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4352.4 | uss_MB: 3574.2 | child_rss_MB: 3998.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 13:16:25,214] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 149.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:04,228] INFO : Step 17200/50000 rank0 | Train Loss: 9.0451 | Train Loss Unweighted: 2.9714 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.6594 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1312 | Regression_eta: 0.0122 | Regression_pt: 0.1290 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8969 | Classification_binary: 2.6594 | Regression_cos_phi: 0.6266 | Regression_energy: 1.8137 | Regression_eta: 0.5999 | Regression_pt: 1.9402 | Regression_sin_phi: 0.5085 | LR: 7.62e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4351s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1094 | max_loss: 0.0032 | max_backward: 0.2028 | max_optimizer_step: 0.1212 | max_tensorboard_logging: 0.2740 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4215.2 | uss_MB: 3435.7 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:17:04,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:17:44,164] INFO : Step 17300/50000 rank0 | Train Loss: 9.1159 | Train Loss Unweighted: 3.0300 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.7018 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1408 | Regression_eta: 0.0111 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9115 | Classification_binary: 2.7018 | Regression_cos_phi: 0.4737 | Regression_energy: 1.9466 | Regression_eta: 0.5473 | Regression_pt: 2.1046 | Regression_sin_phi: 0.4305 | LR: 7.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6047s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0032 | max_backward: 0.3084 | max_optimizer_step: 0.1230 | max_tensorboard_logging: 0.3760 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4215.2 | uss_MB: 3435.8 | child_rss_MB: 4002.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:17:44,377] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:18:23,944] INFO : Step 17400/50000 rank0 | Train Loss: 8.9442 | Train Loss Unweighted: 2.9607 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.6371 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1405 | Regression_eta: 0.0087 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8079 | Classification_binary: 2.6371 | Regression_cos_phi: 0.5797 | Regression_energy: 1.9432 | Regression_eta: 0.4272 | Regression_pt: 2.0788 | Regression_sin_phi: 0.4703 | LR: 7.57e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5094s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0419 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.1613 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4215.2 | uss_MB: 3435.9 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:18:24,170] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:03,992] INFO : Step 17500/50000 rank0 | Train Loss: 8.2494 | Train Loss Unweighted: 2.6931 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.3967 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1260 | Regression_eta: 0.0083 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8651 | Classification_binary: 2.3967 | Regression_cos_phi: 0.5367 | Regression_energy: 1.7420 | Regression_eta: 0.4068 | Regression_pt: 1.9035 | Regression_sin_phi: 0.3986 | LR: 7.54e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.4447s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0937 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.1212 | max_tensorboard_logging: 0.2878 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4215.2 | uss_MB: 3435.9 | child_rss_MB: 4000.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:19:04,208] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 150.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:19:44,089] INFO : Step 17600/50000 rank0 | Train Loss: 9.0382 | Train Loss Unweighted: 2.8387 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5315 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1289 | Regression_eta: 0.0114 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8783 | Classification_binary: 2.5315 | Regression_cos_phi: 0.7775 | Regression_energy: 1.7831 | Regression_eta: 0.5614 | Regression_pt: 1.8434 | Regression_sin_phi: 0.6629 | LR: 7.52e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5366s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0420 | max_backward: 0.2027 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.3588 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4215.2 | uss_MB: 3436.0 | child_rss_MB: 4001.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:19:44,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:20:24,194] INFO : Step 17700/50000 rank0 | Train Loss: 8.5508 | Train Loss Unweighted: 2.9042 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6025 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1280 | Regression_eta: 0.0080 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9100 | Classification_binary: 2.6025 | Regression_cos_phi: 0.4957 | Regression_energy: 1.7703 | Regression_eta: 0.3912 | Regression_pt: 1.9365 | Regression_sin_phi: 0.4446 | LR: 7.49e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8385s | Phase Times: max_pre_step_barrier: 0.1029 | max_batch_to_device: 0.0017 | max_forward: 0.1047 | max_loss: 0.0035 | max_backward: 0.3891 | max_optimizer_step: 0.1235 | max_tensorboard_logging: 0.3826 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4216.2 | uss_MB: 3438.4 | child_rss_MB: 4000.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:20:24,406] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:04,815] INFO : Step 17800/50000 rank0 | Train Loss: 8.3597 | Train Loss Unweighted: 2.7624 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4722 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1244 | Regression_eta: 0.0133 | Regression_pt: 0.1180 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8288 | Classification_binary: 2.4722 | Regression_cos_phi: 0.4952 | Regression_energy: 1.7202 | Regression_eta: 0.6532 | Regression_pt: 1.7753 | Regression_sin_phi: 0.4148 | LR: 7.47e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5011s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1816 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1183 | max_tensorboard_logging: 0.2839 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4215.4 | uss_MB: 3435.8 | child_rss_MB: 3997.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:21:05,034] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:21:45,462] INFO : Step 17900/50000 rank0 | Train Loss: 8.4624 | Train Loss Unweighted: 2.8109 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5032 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1330 | Regression_eta: 0.0071 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8499 | Classification_binary: 2.5032 | Regression_cos_phi: 0.4821 | Regression_energy: 1.8388 | Regression_eta: 0.3466 | Regression_pt: 1.9887 | Regression_sin_phi: 0.4531 | LR: 7.44e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5510s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2501 | max_optimizer_step: 0.1199 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4215.4 | uss_MB: 3435.9 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:21:45,679] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,653] INFO : Step 18000/50000 rank0 | Train Loss: 8.3531 | Train Loss Unweighted: 2.7323 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4349 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1263 | Regression_eta: 0.0083 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8068 | Classification_binary: 2.4349 | Regression_cos_phi: 0.5853 | Regression_energy: 1.7465 | Regression_eta: 0.4069 | Regression_pt: 1.9044 | Regression_sin_phi: 0.4683 | LR: 7.42e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5753s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0886 | max_backward: 0.2032 | max_optimizer_step: 0.1203 | max_tensorboard_logging: 0.3618 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4215.4 | uss_MB: 3436.0 | child_rss_MB: 3997.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:22:25,869] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:22:25,914] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-18000.pth +rank0 [2026-10-28 13:22:26,095] INFO : Running validation on rank0 +rank0 [2026-10-28 13:22:26,095] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:22:26,095] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:22:27,394] INFO : saved batch0_epoch18000.parquet +rank0 [2026-10-28 13:22:27,539] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:22:27,669] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:22:27,741] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:22:27,810] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:22:27,877] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:22:27,945] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:22:28,013] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:22:28,127] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:22:28,258] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:22:28,388] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:22:28,458] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:22:28,527] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:22:28,594] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:22:28,664] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:22:28,733] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:22:28,831] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:22:28,837] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:22:28,840] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:22:28,841] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:22:28,844] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:22:28,845] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:22:28,848] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:22:28,848] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:22:28,851] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:22:28,854] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:22:28,855] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:22:28,858] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:22:28,861] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:22:28,863] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:22:28,864] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:22:28,867] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:22:36,303] INFO : VALIDATION | Step=18000/50000 | Train Loss=8.3531 | Valid Loss=8.9329 | Train Loss Unweighted=2.7323 | Valid Loss Unweighted=2.9060 | Stale=0 | ETA=231.2m +rank0 [2026-10-28 13:22:36,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:36,375] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:22:36,378] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:36,593] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:38,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:39,154] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:39,869] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:40,572] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:40,687] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:22:40,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:40,762] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:22:40,765] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:40,850] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:42,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:43,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:43,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:44,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:44,694] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:22:44,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:22:44,774] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:22:44,777] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:22:44,848] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:22:46,654] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:22:47,337] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:22:48,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:22:48,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:22:48,821] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:22:48,828] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:49,080] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:22:49,526] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:50,947] INFO : Plotted jets +rank0 [2026-10-28 13:22:52,536] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:52,568] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:52,794] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:22:53,166] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:54,534] INFO : Plotted jets +rank0 [2026-10-28 13:22:55,646] INFO : Plotted jet ratio +rank0 [2026-10-28 13:22:55,678] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 13:22:55,884] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_18000/clic_edm_qq_pf +rank0 [2026-10-28 13:22:56,296] INFO : Plotted number of elements +rank0 [2026-10-28 13:22:57,671] INFO : Plotted jets +rank0 [2026-10-28 13:22:58,767] INFO : Plotted jet ratio +rank0 [2026-10-28 13:23:39,452] INFO : Step 18100/50000 rank0 | Train Loss: 9.4806 | Train Loss Unweighted: 2.9017 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5625 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1432 | Regression_eta: 0.0145 | Regression_pt: 0.1409 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8596 | Classification_binary: 2.5625 | Regression_cos_phi: 0.7076 | Regression_energy: 1.9805 | Regression_eta: 0.7129 | Regression_pt: 2.1192 | Regression_sin_phi: 0.5382 | LR: 7.39e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.5871s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1302 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2220 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4410.5 | uss_MB: 3630.6 | child_rss_MB: 4000.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:23:39,677] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:19,416] INFO : Step 18200/50000 rank0 | Train Loss: 8.7523 | Train Loss Unweighted: 2.9918 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.6805 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1349 | Regression_eta: 0.0089 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8484 | Classification_binary: 2.6805 | Regression_cos_phi: 0.5074 | Regression_energy: 1.8649 | Regression_eta: 0.4373 | Regression_pt: 1.9904 | Regression_sin_phi: 0.4233 | LR: 7.36e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.5562s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1856 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3032 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4286.3 | uss_MB: 3507.4 | child_rss_MB: 3999.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:24:19,632] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:24:59,233] INFO : Step 18300/50000 rank0 | Train Loss: 8.8529 | Train Loss Unweighted: 2.8591 | Train Loss Components: Classification: 0.0209 | Classification_binary: 2.5424 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1338 | Regression_eta: 0.0110 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8743 | Classification_binary: 2.5424 | Regression_cos_phi: 0.5607 | Regression_energy: 1.8508 | Regression_eta: 0.5426 | Regression_pt: 2.0245 | Regression_sin_phi: 0.4575 | LR: 7.34e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6474s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3086 | max_optimizer_step: 0.1253 | max_tensorboard_logging: 0.3841 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4286.3 | uss_MB: 3508.6 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:24:59,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:25:38,992] INFO : Step 18400/50000 rank0 | Train Loss: 8.4651 | Train Loss Unweighted: 2.8007 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4982 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1287 | Regression_eta: 0.0082 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8587 | Classification_binary: 2.4982 | Regression_cos_phi: 0.5352 | Regression_energy: 1.7792 | Regression_eta: 0.4023 | Regression_pt: 1.9460 | Regression_sin_phi: 0.4455 | LR: 7.31e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1458 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2249 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4286.3 | uss_MB: 3508.6 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:25:39,219] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:18,759] INFO : Step 18500/50000 rank0 | Train Loss: 8.8058 | Train Loss Unweighted: 2.8294 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5058 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1406 | Regression_eta: 0.0089 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.5058 | Regression_cos_phi: 0.5391 | Regression_energy: 1.9449 | Regression_eta: 0.4376 | Regression_pt: 2.0705 | Regression_sin_phi: 0.4576 | LR: 7.29e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5733s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2608 | max_optimizer_step: 0.1239 | max_tensorboard_logging: 0.3759 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 4286.3 | uss_MB: 3508.7 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:26:18,972] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:26:59,520] INFO : Step 18600/50000 rank0 | Train Loss: 9.6734 | Train Loss Unweighted: 2.9094 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.5557 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1510 | Regression_eta: 0.0108 | Regression_pt: 0.1482 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9224 | Classification_binary: 2.5557 | Regression_cos_phi: 0.7724 | Regression_energy: 2.0885 | Regression_eta: 0.5314 | Regression_pt: 2.2291 | Regression_sin_phi: 0.5738 | LR: 7.26e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6526s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3104 | max_optimizer_step: 0.1236 | max_tensorboard_logging: 0.3941 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4286.3 | uss_MB: 3508.8 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:26:59,733] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:27:39,022] INFO : Step 18700/50000 rank0 | Train Loss: 8.9254 | Train Loss Unweighted: 2.9527 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6460 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1291 | Regression_eta: 0.0112 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8242 | Classification_binary: 2.6460 | Regression_cos_phi: 0.7092 | Regression_energy: 1.7855 | Regression_eta: 0.5524 | Regression_pt: 1.9167 | Regression_sin_phi: 0.4916 | LR: 7.24e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6288s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3072 | max_optimizer_step: 0.1244 | max_tensorboard_logging: 0.3792 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4286.3 | uss_MB: 3508.8 | child_rss_MB: 3999.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:27:39,235] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:19,436] INFO : Step 18800/50000 rank0 | Train Loss: 8.9246 | Train Loss Unweighted: 2.8704 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5517 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1358 | Regression_eta: 0.0098 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8860 | Classification_binary: 2.5517 | Regression_cos_phi: 0.6399 | Regression_energy: 1.8784 | Regression_eta: 0.4827 | Regression_pt: 2.0180 | Regression_sin_phi: 0.4680 | LR: 7.21e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5589s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1158 | max_loss: 0.0034 | max_backward: 0.2031 | max_optimizer_step: 0.2264 | max_tensorboard_logging: 0.2903 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4285.0 | uss_MB: 3507.4 | child_rss_MB: 3997.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:28:19,662] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:28:59,368] INFO : Step 18900/50000 rank0 | Train Loss: 9.1478 | Train Loss Unweighted: 2.9490 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6083 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1479 | Regression_eta: 0.0079 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9106 | Classification_binary: 2.6083 | Regression_cos_phi: 0.5515 | Regression_energy: 2.0450 | Regression_eta: 0.3858 | Regression_pt: 2.2187 | Regression_sin_phi: 0.4279 | LR: 7.18e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5326s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1781 | max_loss: 0.0034 | max_backward: 0.2027 | max_optimizer_step: 0.1219 | max_tensorboard_logging: 0.2977 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4285.0 | uss_MB: 3508.6 | child_rss_MB: 3997.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 98.6 +rank0 [2026-10-28 13:28:59,583] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:40,008] INFO : Step 19000/50000 rank0 | Train Loss: 8.3544 | Train Loss Unweighted: 2.7908 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4964 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1251 | Regression_eta: 0.0091 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7991 | Classification_binary: 2.4964 | Regression_cos_phi: 0.5866 | Regression_energy: 1.7299 | Regression_eta: 0.4479 | Regression_pt: 1.8810 | Regression_sin_phi: 0.4134 | LR: 7.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6320s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3114 | max_optimizer_step: 0.1234 | max_tensorboard_logging: 0.3843 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4285.0 | uss_MB: 3508.7 | child_rss_MB: 4000.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:29:40,221] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:29:40,337] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-19000.pth +rank0 [2026-10-28 13:29:40,515] INFO : Running validation on rank0 +rank0 [2026-10-28 13:29:40,515] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:29:40,515] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:29:41,811] INFO : saved batch0_epoch19000.parquet +rank0 [2026-10-28 13:29:41,954] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:29:42,082] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:29:42,155] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:29:42,223] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:29:42,291] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:29:42,360] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:29:42,428] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:29:42,543] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:29:42,673] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:29:42,803] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:29:42,874] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:29:42,943] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:29:43,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:29:43,079] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:29:43,148] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:29:43,247] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:29:43,253] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:29:43,256] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:29:43,257] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:29:43,260] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:29:43,263] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:29:43,266] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:29:43,267] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:29:43,270] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:29:43,273] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:29:43,275] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:29:43,276] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:29:43,279] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:29:43,282] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:29:43,284] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:29:43,285] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:29:50,636] INFO : VALIDATION | Step=19000/50000 | Train Loss=8.3544 | Valid Loss=8.8895 | Train Loss Unweighted=2.7908 | Valid Loss Unweighted=2.8965 | Stale=0 | ETA=224.0m +rank0 [2026-10-28 13:29:50,646] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:50,756] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:29:50,759] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:29:50,922] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:29:52,901] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:29:53,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:29:54,297] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:29:55,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:29:55,128] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:29:55,131] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:55,189] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:29:55,192] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:29:55,288] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:29:57,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:29:57,802] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:29:58,517] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:29:59,112] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:29:59,240] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:29:59,243] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:29:59,302] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:29:59,305] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:29:59,400] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:30:01,347] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:30:01,920] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:30:02,571] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:30:03,242] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:30:03,370] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:30:03,391] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:30:03,643] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:30:04,090] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:05,505] INFO : Plotted jets +rank0 [2026-10-28 13:30:06,604] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:06,636] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:30:06,858] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:30:07,737] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:09,104] INFO : Plotted jets +rank0 [2026-10-28 13:30:10,213] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:10,247] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 13:30:10,465] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_19000/clic_edm_qq_pf +rank0 [2026-10-28 13:30:10,876] INFO : Plotted number of elements +rank0 [2026-10-28 13:30:12,252] INFO : Plotted jets +rank0 [2026-10-28 13:30:13,344] INFO : Plotted jet ratio +rank0 [2026-10-28 13:30:53,856] INFO : Step 19100/50000 rank0 | Train Loss: 8.9014 | Train Loss Unweighted: 2.9570 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6472 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1308 | Regression_eta: 0.0114 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.6472 | Regression_cos_phi: 0.6696 | Regression_energy: 1.8093 | Regression_eta: 0.5586 | Regression_pt: 1.9562 | Regression_sin_phi: 0.4015 | LR: 7.13e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.1290 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2243 | max_tensorboard_logging: 0.2723 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4476.7 | uss_MB: 3701.8 | child_rss_MB: 3999.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:30:54,081] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:31:33,954] INFO : Step 19200/50000 rank0 | Train Loss: 9.1562 | Train Loss Unweighted: 2.8450 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.5221 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1348 | Regression_eta: 0.0126 | Regression_pt: 0.1336 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9288 | Classification_binary: 2.5221 | Regression_cos_phi: 0.6249 | Regression_energy: 1.8636 | Regression_eta: 0.6214 | Regression_pt: 2.0106 | Regression_sin_phi: 0.5847 | LR: 7.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5058s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0309 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2027 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 4357.1 | uss_MB: 3581.3 | child_rss_MB: 3994.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:31:34,180] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:14,453] INFO : Step 19300/50000 rank0 | Train Loss: 8.7028 | Train Loss Unweighted: 2.7954 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4835 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1315 | Regression_eta: 0.0101 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7829 | Classification_binary: 2.4835 | Regression_cos_phi: 0.6052 | Regression_energy: 1.8183 | Regression_eta: 0.4959 | Regression_pt: 2.0108 | Regression_sin_phi: 0.5061 | LR: 7.08e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5605s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.2189 | max_optimizer_step: 0.1291 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0094 | Memory: rss_MB: 4357.1 | uss_MB: 3582.6 | child_rss_MB: 3992.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:14,673] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:32:54,659] INFO : Step 19400/50000 rank0 | Train Loss: 8.1865 | Train Loss Unweighted: 2.8038 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5172 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1227 | Regression_eta: 0.0086 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8191 | Classification_binary: 2.5172 | Regression_cos_phi: 0.4854 | Regression_energy: 1.6970 | Regression_eta: 0.4226 | Regression_pt: 1.8203 | Regression_sin_phi: 0.4248 | LR: 7.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6244s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3108 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3886 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3991.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:32:54,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:33:34,861] INFO : Step 19500/50000 rank0 | Train Loss: 9.9376 | Train Loss Unweighted: 2.9532 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.5764 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1631 | Regression_eta: 0.0103 | Regression_pt: 0.1618 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8831 | Classification_binary: 2.5764 | Regression_cos_phi: 0.7547 | Regression_energy: 2.2552 | Regression_eta: 0.5084 | Regression_pt: 2.4339 | Regression_sin_phi: 0.5259 | LR: 7.02e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5725s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1802 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1247 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3990.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 13:33:35,075] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:34:14,737] INFO : Step 19600/50000 rank0 | Train Loss: 8.8084 | Train Loss Unweighted: 2.8028 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4812 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1387 | Regression_eta: 0.0093 | Regression_pt: 0.1363 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8578 | Classification_binary: 2.4812 | Regression_cos_phi: 0.5619 | Regression_energy: 1.9174 | Regression_eta: 0.4593 | Regression_pt: 2.0513 | Regression_sin_phi: 0.4797 | LR: 7.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6506s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0034 | max_backward: 0.3053 | max_optimizer_step: 0.1255 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4357.1 | uss_MB: 3582.9 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:14,952] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:34:55,425] INFO : Step 19700/50000 rank0 | Train Loss: 8.4243 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4064 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1219 | Regression_eta: 0.0107 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8412 | Classification_binary: 2.4064 | Regression_cos_phi: 0.6379 | Regression_energy: 1.6860 | Regression_eta: 0.5280 | Regression_pt: 1.8354 | Regression_sin_phi: 0.4894 | LR: 6.97e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1848 | max_loss: 0.0033 | max_backward: 0.2028 | max_optimizer_step: 0.1262 | max_tensorboard_logging: 0.2919 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4357.1 | uss_MB: 3582.8 | child_rss_MB: 3997.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:34:55,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:35:35,274] INFO : Step 19800/50000 rank0 | Train Loss: 8.5778 | Train Loss Unweighted: 2.6571 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3471 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1314 | Regression_eta: 0.0111 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8492 | Classification_binary: 2.3471 | Regression_cos_phi: 0.5786 | Regression_energy: 1.8178 | Regression_eta: 0.5460 | Regression_pt: 1.9564 | Regression_sin_phi: 0.4827 | LR: 6.94e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6656s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3100 | max_optimizer_step: 0.1249 | max_tensorboard_logging: 0.4001 | max_tensorboard_flush: 0.0116 | Memory: rss_MB: 4356.0 | uss_MB: 3581.2 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:35:35,487] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:15,056] INFO : Step 19900/50000 rank0 | Train Loss: 9.2925 | Train Loss Unweighted: 2.8592 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.5260 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1403 | Regression_eta: 0.0102 | Regression_pt: 0.1402 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8466 | Classification_binary: 2.5260 | Regression_cos_phi: 0.6866 | Regression_energy: 1.9402 | Regression_eta: 0.5031 | Regression_pt: 2.1098 | Regression_sin_phi: 0.6802 | LR: 6.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5665s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1028 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.2212 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4357.0 | uss_MB: 3582.5 | child_rss_MB: 3994.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:36:15,285] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:54,856] INFO : Step 20000/50000 rank0 | Train Loss: 8.6915 | Train Loss Unweighted: 2.9342 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1303 | Regression_eta: 0.0098 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9110 | Classification_binary: 2.6305 | Regression_cos_phi: 0.5273 | Regression_energy: 1.8015 | Regression_eta: 0.4812 | Regression_pt: 1.8987 | Regression_sin_phi: 0.4413 | LR: 6.89e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.7457s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0016 | max_forward: 0.1047 | max_loss: 0.0034 | max_backward: 0.3928 | max_optimizer_step: 0.1232 | max_tensorboard_logging: 0.3828 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4357.0 | uss_MB: 3585.1 | child_rss_MB: 3994.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 13:36:55,071] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:36:55,150] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-20000.pth +rank0 [2026-10-28 13:36:55,328] INFO : Running validation on rank0 +rank0 [2026-10-28 13:36:55,328] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:36:55,328] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:36:56,753] INFO : saved batch0_epoch20000.parquet +rank0 [2026-10-28 13:36:56,896] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:36:57,026] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:36:57,098] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:36:57,167] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:36:57,235] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:36:57,303] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:36:57,372] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:36:57,484] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:36:57,614] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:36:57,743] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:36:57,813] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:36:57,881] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:36:57,948] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:36:58,017] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:36:58,086] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:36:58,203] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:36:58,210] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:36:58,212] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:36:58,214] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:36:58,216] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:36:58,219] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:36:58,220] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:36:58,222] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:36:58,223] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:36:58,226] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:36:58,229] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:36:58,231] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:36:58,234] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:36:58,236] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:36:58,238] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:36:58,241] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:37:05,689] INFO : VALIDATION | Step=20000/50000 | Train Loss=8.6915 | Valid Loss=8.7360 | Train Loss Unweighted=2.9342 | Valid Loss Unweighted=2.8415 | Stale=0 | ETA=216.8m +rank0 [2026-10-28 13:37:05,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:05,772] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:37:05,775] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:05,936] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:07,736] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:08,409] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:09,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:09,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:09,980] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:37:09,983] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:10,087] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:37:10,090] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:10,258] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:12,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:12,750] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:13,362] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:14,044] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:14,168] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:37:14,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:37:14,243] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:37:14,246] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:37:14,323] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:37:16,160] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:37:16,739] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:37:17,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:37:18,060] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:37:18,177] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:37:18,185] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:18,437] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:37:18,883] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:20,302] INFO : Plotted jets +rank0 [2026-10-28 13:37:21,912] INFO : Plotted jet ratio +rank0 [2026-10-28 13:37:21,945] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:22,169] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:37:22,543] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:23,914] INFO : Plotted jets +rank0 [2026-10-28 13:37:25,024] INFO : Plotted jet ratio +rank0 [2026-10-28 13:37:25,055] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 13:37:25,258] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_20000/clic_edm_qq_pf +rank0 [2026-10-28 13:37:25,671] INFO : Plotted number of elements +rank0 [2026-10-28 13:37:27,045] INFO : Plotted jets +rank0 [2026-10-28 13:37:28,133] INFO : Plotted jet ratio +rank0 [2026-10-28 13:38:08,072] INFO : Step 20100/50000 rank0 | Train Loss: 8.6005 | Train Loss Unweighted: 2.8225 | Train Loss Components: Classification: 0.0217 | Classification_binary: 2.5255 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1248 | Regression_eta: 0.0112 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9079 | Classification_binary: 2.5255 | Regression_cos_phi: 0.5549 | Regression_energy: 1.7252 | Regression_eta: 0.5490 | Regression_pt: 1.8418 | Regression_sin_phi: 0.4962 | LR: 6.86e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6229s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1364 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2221 | max_tensorboard_logging: 0.2873 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4565.3 | uss_MB: 3791.9 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:38:08,296] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:38:48,547] INFO : Step 20200/50000 rank0 | Train Loss: 8.4530 | Train Loss Unweighted: 2.8603 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.5777 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1186 | Regression_eta: 0.0103 | Regression_pt: 0.1154 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7888 | Classification_binary: 2.5777 | Regression_cos_phi: 0.6280 | Regression_energy: 1.6399 | Regression_eta: 0.5070 | Regression_pt: 1.7360 | Regression_sin_phi: 0.5756 | LR: 6.84e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6168s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.1040 | max_backward: 0.2037 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4430.6 | uss_MB: 3655.1 | child_rss_MB: 3997.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:38:48,762] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:39:28,657] INFO : Step 20300/50000 rank0 | Train Loss: 8.7180 | Train Loss Unweighted: 2.8173 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.5043 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1331 | Regression_eta: 0.0089 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.5043 | Regression_cos_phi: 0.5367 | Regression_energy: 1.8405 | Regression_eta: 0.4382 | Regression_pt: 1.9995 | Regression_sin_phi: 0.5314 | LR: 6.81e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5286s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1857 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.2825 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:39:28,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:08,604] INFO : Step 20400/50000 rank0 | Train Loss: 8.6127 | Train Loss Unweighted: 2.7832 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4907 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1220 | Regression_eta: 0.0134 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8037 | Classification_binary: 2.4907 | Regression_cos_phi: 0.6269 | Regression_energy: 1.6873 | Regression_eta: 0.6593 | Regression_pt: 1.7870 | Regression_sin_phi: 0.5578 | LR: 6.78e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3105 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3924 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 13:40:08,819] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:40:49,925] INFO : Step 20500/50000 rank0 | Train Loss: 8.2046 | Train Loss Unweighted: 2.7946 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.5021 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1247 | Regression_eta: 0.0086 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7926 | Classification_binary: 2.5021 | Regression_cos_phi: 0.4341 | Regression_energy: 1.7240 | Regression_eta: 0.4217 | Regression_pt: 1.9006 | Regression_sin_phi: 0.4297 | LR: 6.76e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6459s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3107 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3769 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4431.6 | uss_MB: 3656.7 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:40:50,141] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:41:30,416] INFO : Step 20600/50000 rank0 | Train Loss: 8.5603 | Train Loss Unweighted: 2.7819 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4715 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1332 | Regression_eta: 0.0089 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8619 | Classification_binary: 2.4715 | Regression_cos_phi: 0.5378 | Regression_energy: 1.8417 | Regression_eta: 0.4356 | Regression_pt: 1.9911 | Regression_sin_phi: 0.4206 | LR: 6.73e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6845s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2126 | max_loss: 0.0035 | max_backward: 0.2874 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 0.2783 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4431.6 | uss_MB: 3659.1 | child_rss_MB: 3993.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 13:41:30,643] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:11,175] INFO : Step 20700/50000 rank0 | Train Loss: 9.3085 | Train Loss Unweighted: 2.8955 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5585 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1420 | Regression_eta: 0.0108 | Regression_pt: 0.1436 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8939 | Classification_binary: 2.5585 | Regression_cos_phi: 0.6888 | Regression_energy: 1.9638 | Regression_eta: 0.5295 | Regression_pt: 2.1598 | Regression_sin_phi: 0.5142 | LR: 6.70e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3116 | max_optimizer_step: 0.1269 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4431.6 | uss_MB: 3657.1 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:11,388] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:42:51,820] INFO : Step 20800/50000 rank0 | Train Loss: 9.4649 | Train Loss Unweighted: 2.9083 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5562 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1514 | Regression_eta: 0.0115 | Regression_pt: 0.1496 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9422 | Classification_binary: 2.5562 | Regression_cos_phi: 0.5871 | Regression_energy: 2.0941 | Regression_eta: 0.5645 | Regression_pt: 2.2511 | Regression_sin_phi: 0.4696 | LR: 6.67e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3090 | max_optimizer_step: 0.1272 | max_tensorboard_logging: 0.3940 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4431.1 | uss_MB: 3655.0 | child_rss_MB: 3994.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:42:52,033] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:43:32,012] INFO : Step 20900/50000 rank0 | Train Loss: 8.7815 | Train Loss Unweighted: 2.7980 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1329 | Regression_eta: 0.0109 | Regression_pt: 0.1284 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8653 | Classification_binary: 2.4868 | Regression_cos_phi: 0.5314 | Regression_energy: 1.8383 | Regression_eta: 0.5345 | Regression_pt: 1.9316 | Regression_sin_phi: 0.5936 | LR: 6.65e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5397s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1787 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.1227 | max_tensorboard_logging: 0.2914 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4433.1 | uss_MB: 3656.4 | child_rss_MB: 3993.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 13:43:32,225] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:11,928] INFO : Step 21000/50000 rank0 | Train Loss: 8.7870 | Train Loss Unweighted: 2.9215 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.6127 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1299 | Regression_eta: 0.0101 | Regression_pt: 0.1315 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8406 | Classification_binary: 2.6127 | Regression_cos_phi: 0.5266 | Regression_energy: 1.7959 | Regression_eta: 0.4972 | Regression_pt: 1.9782 | Regression_sin_phi: 0.5358 | LR: 6.62e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6279s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3086 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4433.1 | uss_MB: 3656.7 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 13:44:12,143] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:44:12,257] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-21000.pth +rank0 [2026-10-28 13:44:12,433] INFO : Running validation on rank0 +rank0 [2026-10-28 13:44:12,433] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:44:12,433] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:44:13,723] INFO : saved batch0_epoch21000.parquet +rank0 [2026-10-28 13:44:13,866] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:44:13,997] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:44:14,070] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:44:14,138] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:44:14,206] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:44:14,276] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:44:14,345] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:44:14,458] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:44:14,588] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:44:14,719] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:44:14,789] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:44:14,859] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:44:14,927] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:44:14,995] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:44:15,064] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:44:15,160] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:44:15,166] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:44:15,169] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:44:15,170] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:44:15,172] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:44:15,173] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:44:15,176] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:44:15,177] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:44:15,180] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:44:15,182] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:44:15,183] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:44:15,186] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:44:15,187] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:44:15,190] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:44:15,192] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:44:15,195] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:44:22,632] INFO : VALIDATION | Step=21000/50000 | Train Loss=8.7870 | Valid Loss=8.7762 | Train Loss Unweighted=2.9215 | Valid Loss Unweighted=2.8587 | Stale=1 | ETA=209.7m +rank0 [2026-10-28 13:44:22,642] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:22,725] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:44:22,728] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:22,889] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:24,865] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:25,489] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:26,165] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:26,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:26,973] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:44:26,976] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:27,032] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:44:27,035] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:27,130] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:29,036] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:29,665] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:30,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:30,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:31,050] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:44:31,054] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:44:31,109] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:44:31,112] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:44:31,205] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:44:32,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:44:33,684] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:44:34,377] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:44:35,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:44:35,199] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:44:35,207] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:35,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:44:35,904] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:37,325] INFO : Plotted jets +rank0 [2026-10-28 13:44:38,954] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:38,990] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:39,213] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:44:39,589] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:40,964] INFO : Plotted jets +rank0 [2026-10-28 13:44:42,076] INFO : Plotted jet ratio +rank0 [2026-10-28 13:44:42,108] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 13:44:42,314] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_21000/clic_edm_qq_pf +rank0 [2026-10-28 13:44:42,726] INFO : Plotted number of elements +rank0 [2026-10-28 13:44:44,111] INFO : Plotted jets +rank0 [2026-10-28 13:44:45,202] INFO : Plotted jet ratio +rank0 [2026-10-28 13:45:25,044] INFO : Step 21100/50000 rank0 | Train Loss: 9.1603 | Train Loss Unweighted: 2.9861 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.6502 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1451 | Regression_eta: 0.0080 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.6502 | Regression_cos_phi: 0.5427 | Regression_energy: 2.0070 | Regression_eta: 0.3926 | Regression_pt: 2.1796 | Regression_sin_phi: 0.5291 | LR: 6.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.4806s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0099 | max_forward: 0.0780 | max_loss: 0.0031 | max_backward: 0.2030 | max_optimizer_step: 0.1540 | max_tensorboard_logging: 0.2848 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4647.8 | uss_MB: 3871.0 | child_rss_MB: 3991.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:45:25,273] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 148.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:45:28,646] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-10-28 13:45:28,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 13:46:04,784] INFO : Step 21200/50000 rank0 | Train Loss: 8.6887 | Train Loss Unweighted: 2.8540 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.5406 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1346 | Regression_eta: 0.0089 | Regression_pt: 0.1332 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8919 | Classification_binary: 2.5406 | Regression_cos_phi: 0.5094 | Regression_energy: 1.8607 | Regression_eta: 0.4376 | Regression_pt: 2.0043 | Regression_sin_phi: 0.4442 | LR: 6.56e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.5521s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0124 | max_backward: 0.2026 | max_optimizer_step: 0.1283 | max_tensorboard_logging: 0.3553 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4504.8 | uss_MB: 3725.6 | child_rss_MB: 3989.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 13:46:04,997] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:46:44,984] INFO : Step 21300/50000 rank0 | Train Loss: 8.9032 | Train Loss Unweighted: 2.8642 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5416 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1399 | Regression_eta: 0.0101 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9045 | Classification_binary: 2.5416 | Regression_cos_phi: 0.4843 | Regression_energy: 1.9340 | Regression_eta: 0.4975 | Regression_pt: 2.0265 | Regression_sin_phi: 0.5149 | LR: 6.54e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5908s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1046 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2231 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4505.8 | uss_MB: 3727.3 | child_rss_MB: 3989.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:46:45,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:47:24,692] INFO : Step 21400/50000 rank0 | Train Loss: 8.8688 | Train Loss Unweighted: 2.7766 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4588 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1337 | Regression_eta: 0.0118 | Regression_pt: 0.1335 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8862 | Classification_binary: 2.4588 | Regression_cos_phi: 0.5479 | Regression_energy: 1.8489 | Regression_eta: 0.5782 | Regression_pt: 2.0087 | Regression_sin_phi: 0.5402 | LR: 6.51e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6506s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.2797 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3911 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4505.8 | uss_MB: 3727.5 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:47:24,905] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:04,093] INFO : Step 21500/50000 rank0 | Train Loss: 8.5595 | Train Loss Unweighted: 2.8174 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5183 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1274 | Regression_eta: 0.0105 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8405 | Classification_binary: 2.5183 | Regression_cos_phi: 0.5454 | Regression_energy: 1.7619 | Regression_eta: 0.5166 | Regression_pt: 1.8647 | Regression_sin_phi: 0.5121 | LR: 6.48e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5571s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0586 | max_backward: 0.2031 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3442 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4505.8 | uss_MB: 3727.7 | child_rss_MB: 3992.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:48:04,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:48:43,303] INFO : Step 21600/50000 rank0 | Train Loss: 8.8485 | Train Loss Unweighted: 2.7954 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4770 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1346 | Regression_eta: 0.0090 | Regression_pt: 0.1355 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8574 | Classification_binary: 2.4770 | Regression_cos_phi: 0.6364 | Regression_energy: 1.8608 | Regression_eta: 0.4430 | Regression_pt: 2.0383 | Regression_sin_phi: 0.5356 | LR: 6.45e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5861s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.2532 | max_optimizer_step: 0.1260 | max_tensorboard_logging: 0.3750 | max_tensorboard_flush: 0.0146 | Memory: rss_MB: 4505.8 | uss_MB: 3727.8 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:48:43,520] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:49:23,335] INFO : Step 21700/50000 rank0 | Train Loss: 8.2277 | Train Loss Unweighted: 2.7442 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4444 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1309 | Regression_eta: 0.0082 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7969 | Classification_binary: 2.4444 | Regression_cos_phi: 0.4774 | Regression_energy: 1.8100 | Regression_eta: 0.4050 | Regression_pt: 1.9270 | Regression_sin_phi: 0.3669 | LR: 6.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2595 | max_optimizer_step: 0.1245 | max_tensorboard_logging: 0.3818 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4505.8 | uss_MB: 3727.8 | child_rss_MB: 3993.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:49:23,548] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:03,504] INFO : Step 21800/50000 rank0 | Train Loss: 8.2401 | Train Loss Unweighted: 2.7430 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4539 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1220 | Regression_eta: 0.0080 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7828 | Classification_binary: 2.4539 | Regression_cos_phi: 0.5258 | Regression_energy: 1.6873 | Regression_eta: 0.3944 | Regression_pt: 1.8496 | Regression_sin_phi: 0.5464 | LR: 6.40e-04 | DataLoad Time: 0.0027s | Model Forward Time: 1.6512s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0033 | max_backward: 0.3088 | max_optimizer_step: 0.1251 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4505.2 | uss_MB: 3725.7 | child_rss_MB: 3995.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:50:03,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:50:43,587] INFO : Step 21900/50000 rank0 | Train Loss: 8.4682 | Train Loss Unweighted: 2.8525 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.5576 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1265 | Regression_eta: 0.0103 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8414 | Classification_binary: 2.5576 | Regression_cos_phi: 0.4868 | Regression_energy: 1.7487 | Regression_eta: 0.5044 | Regression_pt: 1.8383 | Regression_sin_phi: 0.4909 | LR: 6.37e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5877s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.2617 | max_optimizer_step: 0.1225 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4506.2 | uss_MB: 3727.2 | child_rss_MB: 3992.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:50:43,801] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:24,156] INFO : Step 22000/50000 rank0 | Train Loss: 9.2807 | Train Loss Unweighted: 2.8615 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.5113 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1525 | Regression_eta: 0.0099 | Regression_pt: 0.1497 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9062 | Classification_binary: 2.5113 | Regression_cos_phi: 0.5237 | Regression_energy: 2.1090 | Regression_eta: 0.4875 | Regression_pt: 2.2517 | Regression_sin_phi: 0.4914 | LR: 6.34e-04 | DataLoad Time: 0.0030s | Model Forward Time: 1.6164s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3044 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4506.2 | uss_MB: 3727.4 | child_rss_MB: 3999.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:51:24,368] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:51:24,386] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-22000.pth +rank0 [2026-10-28 13:51:24,671] INFO : Running validation on rank0 +rank0 [2026-10-28 13:51:24,671] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:51:24,671] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:51:25,865] INFO : saved batch0_epoch22000.parquet +rank0 [2026-10-28 13:51:26,010] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:51:26,140] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:51:26,214] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:51:26,282] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:51:26,351] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:51:26,420] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:51:26,488] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:51:26,600] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:51:26,730] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:51:26,861] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:51:26,932] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:51:27,000] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:51:27,068] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:51:27,137] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:51:27,205] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:51:27,316] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:51:27,409] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:51:27,412] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:51:27,414] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:51:27,417] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:51:27,418] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:51:27,421] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:51:27,424] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:51:27,425] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:51:27,427] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:51:27,428] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:51:27,431] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:51:27,434] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:51:27,437] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:51:27,437] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:51:27,440] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:51:34,881] INFO : VALIDATION | Step=22000/50000 | Train Loss=9.2807 | Valid Loss=8.6936 | Train Loss Unweighted=2.8615 | Valid Loss Unweighted=2.8315 | Stale=0 | ETA=202.4m +rank0 [2026-10-28 13:51:34,890] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:34,952] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:51:34,955] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:35,032] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:37,028] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:37,693] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:38,387] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:39,056] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:39,168] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:39,171] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:39,227] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:51:39,230] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:39,328] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:41,144] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:41,818] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:42,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:43,172] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:43,317] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:43,321] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:51:43,375] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:51:43,378] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:51:43,481] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:51:45,414] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:51:45,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:51:46,637] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:51:47,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:51:47,460] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:51:47,468] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:47,717] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:51:48,173] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:49,588] INFO : Plotted jets +rank0 [2026-10-28 13:51:51,187] INFO : Plotted jet ratio +rank0 [2026-10-28 13:51:51,219] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:51,440] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:51:51,812] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:53,189] INFO : Plotted jets +rank0 [2026-10-28 13:51:54,300] INFO : Plotted jet ratio +rank0 [2026-10-28 13:51:54,333] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 13:51:54,537] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_22000/clic_edm_qq_pf +rank0 [2026-10-28 13:51:54,947] INFO : Plotted number of elements +rank0 [2026-10-28 13:51:56,329] INFO : Plotted jets +rank0 [2026-10-28 13:51:57,416] INFO : Plotted jet ratio +rank0 [2026-10-28 13:52:37,740] INFO : Step 22100/50000 rank0 | Train Loss: 7.6891 | Train Loss Unweighted: 2.6576 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3845 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1172 | Regression_eta: 0.0077 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7739 | Classification_binary: 2.3845 | Regression_cos_phi: 0.4335 | Regression_energy: 1.6213 | Regression_eta: 0.3801 | Regression_pt: 1.7640 | Regression_sin_phi: 0.3317 | LR: 6.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6150s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1368 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2167 | max_tensorboard_logging: 0.2992 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4712.8 | uss_MB: 3935.8 | child_rss_MB: 3993.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:52:37,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:17,798] INFO : Step 22200/50000 rank0 | Train Loss: 8.0657 | Train Loss Unweighted: 2.6653 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.3825 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1190 | Regression_eta: 0.0093 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8310 | Classification_binary: 2.3825 | Regression_cos_phi: 0.4882 | Regression_energy: 1.6456 | Regression_eta: 0.4580 | Regression_pt: 1.7917 | Regression_sin_phi: 0.4687 | LR: 6.29e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7037s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1063 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3114 | max_optimizer_step: 0.1305 | max_tensorboard_logging: 0.2814 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4712.8 | uss_MB: 3935.8 | child_rss_MB: 4016.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:53:18,014] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:53:57,684] INFO : Step 22300/50000 rank0 | Train Loss: 8.9171 | Train Loss Unweighted: 2.8580 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5284 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1419 | Regression_eta: 0.0093 | Regression_pt: 0.1421 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.5284 | Regression_cos_phi: 0.4968 | Regression_energy: 1.9618 | Regression_eta: 0.4559 | Regression_pt: 2.1379 | Regression_sin_phi: 0.4849 | LR: 6.26e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6013s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1277 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.2222 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 4582.3 | uss_MB: 3806.6 | child_rss_MB: 3996.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:53:57,914] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:54:38,036] INFO : Step 22400/50000 rank0 | Train Loss: 8.3987 | Train Loss Unweighted: 2.8351 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.5368 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1292 | Regression_eta: 0.0076 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 2.5368 | Regression_cos_phi: 0.5283 | Regression_energy: 1.7865 | Regression_eta: 0.3747 | Regression_pt: 1.8937 | Regression_sin_phi: 0.4508 | LR: 6.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6479s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.3046 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3755 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4582.3 | uss_MB: 3807.0 | child_rss_MB: 3992.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:54:38,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:18,039] INFO : Step 22500/50000 rank0 | Train Loss: 7.9802 | Train Loss Unweighted: 2.8337 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.5668 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1130 | Regression_eta: 0.0084 | Regression_pt: 0.1106 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8215 | Classification_binary: 2.5668 | Regression_cos_phi: 0.5451 | Regression_energy: 1.5633 | Regression_eta: 0.4110 | Regression_pt: 1.6641 | Regression_sin_phi: 0.4085 | LR: 6.20e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6357s | Phase Times: max_pre_step_barrier: 0.0004 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0031 | max_backward: 0.3038 | max_optimizer_step: 0.1254 | max_tensorboard_logging: 0.3747 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4582.3 | uss_MB: 3807.2 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:55:18,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:55:57,947] INFO : Step 22600/50000 rank0 | Train Loss: 8.2434 | Train Loss Unweighted: 2.7290 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4346 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1262 | Regression_eta: 0.0091 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8245 | Classification_binary: 2.4346 | Regression_cos_phi: 0.4661 | Regression_energy: 1.7459 | Regression_eta: 0.4494 | Regression_pt: 1.8718 | Regression_sin_phi: 0.4512 | LR: 6.18e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5957s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0034 | max_backward: 0.2477 | max_optimizer_step: 0.1288 | max_tensorboard_logging: 0.3796 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4582.3 | uss_MB: 3807.2 | child_rss_MB: 3995.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:55:58,162] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:56:38,034] INFO : Step 22700/50000 rank0 | Train Loss: 8.7923 | Train Loss Unweighted: 2.8562 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5564 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1238 | Regression_eta: 0.0105 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8508 | Classification_binary: 2.5564 | Regression_cos_phi: 0.6522 | Regression_energy: 1.7123 | Regression_eta: 0.5148 | Regression_pt: 1.8679 | Regression_sin_phi: 0.6378 | LR: 6.15e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1833 | max_loss: 0.0034 | max_backward: 0.2036 | max_optimizer_step: 0.1231 | max_tensorboard_logging: 0.2861 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4582.3 | uss_MB: 3807.3 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:56:38,248] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:18,120] INFO : Step 22800/50000 rank0 | Train Loss: 8.1902 | Train Loss Unweighted: 2.7412 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4551 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1205 | Regression_eta: 0.0079 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8718 | Classification_binary: 2.4551 | Regression_cos_phi: 0.5672 | Regression_energy: 1.6660 | Regression_eta: 0.3884 | Regression_pt: 1.8211 | Regression_sin_phi: 0.4206 | LR: 6.12e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6259s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1857 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2252 | max_tensorboard_logging: 0.2721 | max_tensorboard_flush: 0.0093 | Memory: rss_MB: 4582.3 | uss_MB: 3807.4 | child_rss_MB: 3990.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:57:18,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:57:57,776] INFO : Step 22900/50000 rank0 | Train Loss: 8.5141 | Train Loss Unweighted: 2.7533 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4376 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1377 | Regression_eta: 0.0100 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.4376 | Regression_cos_phi: 0.5244 | Regression_energy: 1.9043 | Regression_eta: 0.4892 | Regression_pt: 2.0402 | Regression_sin_phi: 0.3413 | LR: 6.09e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5540s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0773 | max_loss: 0.0516 | max_backward: 0.2035 | max_optimizer_step: 0.1238 | max_tensorboard_logging: 0.3405 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 4581.7 | uss_MB: 3806.6 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 13:57:57,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:37,723] INFO : Step 23000/50000 rank0 | Train Loss: 8.0713 | Train Loss Unweighted: 2.6961 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4237 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1154 | Regression_eta: 0.0101 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8141 | Classification_binary: 2.4237 | Regression_cos_phi: 0.6011 | Regression_energy: 1.5958 | Regression_eta: 0.4957 | Regression_pt: 1.6533 | Regression_sin_phi: 0.4875 | LR: 6.06e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0034 | max_backward: 0.2271 | max_optimizer_step: 0.1257 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4581.7 | uss_MB: 3807.1 | child_rss_MB: 3993.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 13:58:37,937] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 13:58:38,047] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-23000.pth +rank0 [2026-10-28 13:58:38,223] INFO : Running validation on rank0 +rank0 [2026-10-28 13:58:38,223] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 13:58:38,223] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 13:58:39,522] INFO : saved batch0_epoch23000.parquet +rank0 [2026-10-28 13:58:39,665] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 13:58:39,795] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 13:58:39,868] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 13:58:39,936] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 13:58:40,004] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 13:58:40,073] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 13:58:40,142] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 13:58:40,256] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 13:58:40,385] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 13:58:40,515] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 13:58:40,586] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 13:58:40,654] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 13:58:40,721] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 13:58:40,789] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 13:58:40,858] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 13:58:40,954] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 13:58:40,960] INFO : plotted pt_target histogram +rank0 [2026-10-28 13:58:40,962] INFO : plotted pt_pred histogram +rank0 [2026-10-28 13:58:40,964] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 13:58:40,965] INFO : plotted eta_target histogram +rank0 [2026-10-28 13:58:40,967] INFO : plotted eta_pred histogram +rank0 [2026-10-28 13:58:40,968] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 13:58:40,971] INFO : plotted sphi_target histogram +rank0 [2026-10-28 13:58:40,974] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 13:58:40,977] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 13:58:40,978] INFO : plotted cphi_target histogram +rank0 [2026-10-28 13:58:40,981] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 13:58:40,983] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 13:58:40,984] INFO : plotted energy_target histogram +rank0 [2026-10-28 13:58:40,986] INFO : plotted energy_pred histogram +rank0 [2026-10-28 13:58:40,989] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 13:58:48,307] INFO : VALIDATION | Step=23000/50000 | Train Loss=8.0713 | Valid Loss=8.6660 | Train Loss Unweighted=2.6961 | Valid Loss Unweighted=2.8138 | Stale=0 | ETA=195.2m +rank0 [2026-10-28 13:58:48,316] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:48,377] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 13:58:48,381] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 13:58:48,456] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:50,189] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:50,820] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:51,447] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 13:58:52,058] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 13:58:52,229] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:58:52,232] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:52,288] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 13:58:52,290] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:58:52,389] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:54,235] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:54,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:55,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 13:58:56,167] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 13:58:56,285] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 13:58:56,288] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 13:58:56,342] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 13:58:56,344] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 13:58:56,451] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 13:58:58,262] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 13:58:58,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 13:58:59,593] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 13:59:00,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 13:59:00,415] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 13:59:00,423] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:59:00,679] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ttbar_pf +rank0 [2026-10-28 13:59:01,126] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:02,548] INFO : Plotted jets +rank0 [2026-10-28 13:59:04,187] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:04,220] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:59:04,444] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 13:59:04,821] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:06,243] INFO : Plotted jets +rank0 [2026-10-28 13:59:07,337] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:07,369] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 13:59:07,569] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_23000/clic_edm_qq_pf +rank0 [2026-10-28 13:59:07,982] INFO : Plotted number of elements +rank0 [2026-10-28 13:59:09,360] INFO : Plotted jets +rank0 [2026-10-28 13:59:10,465] INFO : Plotted jet ratio +rank0 [2026-10-28 13:59:50,823] INFO : Step 23100/50000 rank0 | Train Loss: 8.7210 | Train Loss Unweighted: 2.8490 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.5345 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1341 | Regression_eta: 0.0089 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7954 | Classification_binary: 2.5345 | Regression_cos_phi: 0.5529 | Regression_energy: 1.8547 | Regression_eta: 0.4365 | Regression_pt: 2.0352 | Regression_sin_phi: 0.5118 | LR: 6.04e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5440s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0450 | max_backward: 0.2026 | max_optimizer_step: 0.1312 | max_tensorboard_logging: 0.3097 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4780.1 | uss_MB: 4007.1 | child_rss_MB: 3990.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 13:59:51,035] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:00:30,940] INFO : Step 23200/50000 rank0 | Train Loss: 8.1690 | Train Loss Unweighted: 2.7382 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4544 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1204 | Regression_eta: 0.0096 | Regression_pt: 0.1182 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8525 | Classification_binary: 2.4544 | Regression_cos_phi: 0.5437 | Regression_energy: 1.6648 | Regression_eta: 0.4734 | Regression_pt: 1.7778 | Regression_sin_phi: 0.4024 | LR: 6.01e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1860 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1296 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4780.1 | uss_MB: 4007.1 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:00:31,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:10,778] INFO : Step 23300/50000 rank0 | Train Loss: 8.2610 | Train Loss Unweighted: 2.7374 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4437 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1259 | Regression_eta: 0.0086 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8109 | Classification_binary: 2.4437 | Regression_cos_phi: 0.5753 | Regression_energy: 1.7404 | Regression_eta: 0.4237 | Regression_pt: 1.8694 | Regression_sin_phi: 0.3976 | LR: 5.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5512s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0386 | max_backward: 0.2031 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.3325 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4651.5 | uss_MB: 3878.1 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:01:10,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:01:50,742] INFO : Step 23400/50000 rank0 | Train Loss: 8.4409 | Train Loss Unweighted: 2.6613 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3558 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1302 | Regression_eta: 0.0098 | Regression_pt: 0.1291 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8193 | Classification_binary: 2.3558 | Regression_cos_phi: 0.5466 | Regression_energy: 1.8005 | Regression_eta: 0.4833 | Regression_pt: 1.9422 | Regression_sin_phi: 0.4932 | LR: 5.95e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6048s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2487 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3860 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4651.5 | uss_MB: 3878.3 | child_rss_MB: 3993.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:01:50,953] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:02:30,823] INFO : Step 23500/50000 rank0 | Train Loss: 8.5490 | Train Loss Unweighted: 2.6953 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3944 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1266 | Regression_eta: 0.0134 | Regression_pt: 0.1243 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8170 | Classification_binary: 2.3944 | Regression_cos_phi: 0.5662 | Regression_energy: 1.7513 | Regression_eta: 0.6579 | Regression_pt: 1.8696 | Regression_sin_phi: 0.4925 | LR: 5.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5634s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1824 | max_loss: 0.0036 | max_backward: 0.2037 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.2816 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4651.5 | uss_MB: 3878.4 | child_rss_MB: 3987.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:02:31,042] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:10,252] INFO : Step 23600/50000 rank0 | Train Loss: 8.8357 | Train Loss Unweighted: 2.9482 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.6350 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1346 | Regression_eta: 0.0084 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7946 | Classification_binary: 2.6350 | Regression_cos_phi: 0.6046 | Regression_energy: 1.8609 | Regression_eta: 0.4145 | Regression_pt: 2.0000 | Regression_sin_phi: 0.5261 | LR: 5.90e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5729s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2263 | max_tensorboard_logging: 0.2874 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 4651.5 | uss_MB: 3878.5 | child_rss_MB: 3988.7 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:03:10,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:03:49,941] INFO : Step 23700/50000 rank0 | Train Loss: 8.1938 | Train Loss Unweighted: 2.7824 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4951 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1235 | Regression_eta: 0.0086 | Regression_pt: 0.1199 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8722 | Classification_binary: 2.4951 | Regression_cos_phi: 0.4781 | Regression_energy: 1.7083 | Regression_eta: 0.4236 | Regression_pt: 1.8033 | Regression_sin_phi: 0.4133 | LR: 5.87e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7260s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1043 | max_loss: 0.0034 | max_backward: 0.3810 | max_optimizer_step: 0.1270 | max_tensorboard_logging: 0.3539 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4652.5 | uss_MB: 3880.6 | child_rss_MB: 3990.3 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 84.9 +rank0 [2026-10-28 14:03:50,156] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:04:30,460] INFO : Step 23800/50000 rank0 | Train Loss: 8.6763 | Train Loss Unweighted: 2.8378 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.5286 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1327 | Regression_eta: 0.0087 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8504 | Classification_binary: 2.5286 | Regression_cos_phi: 0.5495 | Regression_energy: 1.8346 | Regression_eta: 0.4268 | Regression_pt: 1.9561 | Regression_sin_phi: 0.5303 | LR: 5.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5155s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0625 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.2761 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4652.5 | uss_MB: 3878.6 | child_rss_MB: 3990.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:04:30,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:10,157] INFO : Step 23900/50000 rank0 | Train Loss: 8.9800 | Train Loss Unweighted: 2.9394 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.6292 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1302 | Regression_eta: 0.0107 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9030 | Classification_binary: 2.6292 | Regression_cos_phi: 0.6787 | Regression_energy: 1.8001 | Regression_eta: 0.5279 | Regression_pt: 1.9368 | Regression_sin_phi: 0.5043 | LR: 5.81e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5496s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0574 | max_backward: 0.2034 | max_optimizer_step: 0.1250 | max_tensorboard_logging: 0.3414 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 4653.0 | uss_MB: 3878.3 | child_rss_MB: 3989.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:05:10,371] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:50,193] INFO : Step 24000/50000 rank0 | Train Loss: 8.8756 | Train Loss Unweighted: 2.7751 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.4565 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1352 | Regression_eta: 0.0103 | Regression_pt: 0.1341 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8351 | Classification_binary: 2.4565 | Regression_cos_phi: 0.7104 | Regression_energy: 1.8692 | Regression_eta: 0.5068 | Regression_pt: 2.0173 | Regression_sin_phi: 0.4803 | LR: 5.78e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1846 | max_loss: 0.0034 | max_backward: 0.2026 | max_optimizer_step: 0.1252 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4653.0 | uss_MB: 3878.5 | child_rss_MB: 3992.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:05:50,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:05:50,428] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-24000.pth +rank0 [2026-10-28 14:05:50,600] INFO : Running validation on rank0 +rank0 [2026-10-28 14:05:50,600] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:05:50,600] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:05:51,891] INFO : saved batch0_epoch24000.parquet +rank0 [2026-10-28 14:05:52,036] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:05:52,165] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:05:52,238] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:05:52,307] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:05:52,375] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:05:52,444] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:05:52,512] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:05:52,625] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:05:52,755] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:05:52,891] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:05:52,961] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:05:53,030] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:05:53,099] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:05:53,167] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:05:53,236] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:05:53,335] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:05:53,342] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:05:53,345] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:05:53,348] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:05:53,349] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:05:53,351] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:05:53,354] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:05:53,355] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:05:53,358] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:05:53,361] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:05:53,364] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:05:53,366] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:05:53,368] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:05:53,370] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:05:53,371] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:05:53,374] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:06:00,801] INFO : VALIDATION | Step=24000/50000 | Train Loss=8.8756 | Valid Loss=8.6014 | Train Loss Unweighted=2.7751 | Valid Loss Unweighted=2.7943 | Stale=0 | ETA=187.9m +rank0 [2026-10-28 14:06:00,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:00,873] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:06:00,876] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:01,041] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:02,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:03,657] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:04,373] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:05,072] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:05,197] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:06:05,200] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:05,256] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:06:05,259] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:05,361] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:07,261] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:07,894] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:08,504] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:09,151] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:09,290] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:06:09,294] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:06:09,349] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:06:09,352] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:06:09,452] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:06:11,228] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:06:11,898] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:06:12,585] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:06:13,258] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:06:13,409] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:06:13,417] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:13,669] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:06:14,116] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:15,536] INFO : Plotted jets +rank0 [2026-10-28 14:06:17,202] INFO : Plotted jet ratio +rank0 [2026-10-28 14:06:17,234] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:17,459] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:06:17,830] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:19,210] INFO : Plotted jets +rank0 [2026-10-28 14:06:20,274] INFO : Plotted jet ratio +rank0 [2026-10-28 14:06:20,306] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 14:06:20,510] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_24000/clic_edm_qq_pf +rank0 [2026-10-28 14:06:20,924] INFO : Plotted number of elements +rank0 [2026-10-28 14:06:22,306] INFO : Plotted jets +rank0 [2026-10-28 14:06:23,419] INFO : Plotted jet ratio +rank0 [2026-10-28 14:07:04,154] INFO : Step 24100/50000 rank0 | Train Loss: 8.7307 | Train Loss Unweighted: 2.7990 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4931 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1296 | Regression_eta: 0.0104 | Regression_pt: 0.1261 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8923 | Classification_binary: 2.4931 | Regression_cos_phi: 0.6236 | Regression_energy: 1.7919 | Regression_eta: 0.5120 | Regression_pt: 1.8972 | Regression_sin_phi: 0.5206 | LR: 5.75e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6853s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0883 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3094 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4855.8 | uss_MB: 4083.5 | child_rss_MB: 4011.4 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:07:04,369] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:07:43,728] INFO : Step 24200/50000 rank0 | Train Loss: 8.0567 | Train Loss Unweighted: 2.7548 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4834 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1142 | Regression_eta: 0.0105 | Regression_pt: 0.1111 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8181 | Classification_binary: 2.4834 | Regression_cos_phi: 0.4659 | Regression_energy: 1.5792 | Regression_eta: 0.5147 | Regression_pt: 1.6710 | Regression_sin_phi: 0.5244 | LR: 5.73e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7035s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3067 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3874 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4855.8 | uss_MB: 4083.5 | child_rss_MB: 3986.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:07:43,939] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:08:23,336] INFO : Step 24300/50000 rank0 | Train Loss: 9.2852 | Train Loss Unweighted: 2.9776 | Train Loss Components: Classification: 0.0211 | Classification_binary: 2.6438 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1437 | Regression_eta: 0.0115 | Regression_pt: 0.1398 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8828 | Classification_binary: 2.6438 | Regression_cos_phi: 0.6426 | Regression_energy: 1.9871 | Regression_eta: 0.5643 | Regression_pt: 2.1033 | Regression_sin_phi: 0.4613 | LR: 5.70e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5554s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0722 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.2033 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4725.2 | uss_MB: 3954.4 | child_rss_MB: 3992.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:08:23,561] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:03,726] INFO : Step 24400/50000 rank0 | Train Loss: 8.1835 | Train Loss Unweighted: 2.8827 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.6032 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1185 | Regression_eta: 0.0096 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7228 | Classification_binary: 2.6032 | Regression_cos_phi: 0.5069 | Regression_energy: 1.6389 | Regression_eta: 0.4740 | Regression_pt: 1.7838 | Regression_sin_phi: 0.4539 | LR: 5.67e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6779s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0034 | max_backward: 0.3087 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3821 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4726.2 | uss_MB: 3954.6 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:09:03,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:09:43,128] INFO : Step 24500/50000 rank0 | Train Loss: 8.0380 | Train Loss Unweighted: 2.6094 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.3305 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1182 | Regression_eta: 0.0097 | Regression_pt: 0.1139 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8908 | Classification_binary: 2.3305 | Regression_cos_phi: 0.5796 | Regression_energy: 1.6340 | Regression_eta: 0.4751 | Regression_pt: 1.7137 | Regression_sin_phi: 0.4144 | LR: 5.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5978s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2277 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4726.2 | uss_MB: 3954.6 | child_rss_MB: 3989.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:09:43,341] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:10:22,389] INFO : Step 24600/50000 rank0 | Train Loss: 8.9178 | Train Loss Unweighted: 2.7763 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4565 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1358 | Regression_eta: 0.0105 | Regression_pt: 0.1340 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8573 | Classification_binary: 2.4565 | Regression_cos_phi: 0.7352 | Regression_energy: 1.8779 | Regression_eta: 0.5148 | Regression_pt: 2.0162 | Regression_sin_phi: 0.4600 | LR: 5.61e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.6649s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3080 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3794 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:10:22,602] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:11:02,736] INFO : Step 24700/50000 rank0 | Train Loss: 8.5762 | Train Loss Unweighted: 2.7821 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4810 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1263 | Regression_eta: 0.0090 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.4810 | Regression_cos_phi: 0.5978 | Regression_energy: 1.7459 | Regression_eta: 0.4423 | Regression_pt: 1.9101 | Regression_sin_phi: 0.5477 | LR: 5.58e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0847 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.2884 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3989.3 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:11:02,959] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:11:42,230] INFO : Step 24800/50000 rank0 | Train Loss: 8.5547 | Train Loss Unweighted: 2.7416 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4371 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1283 | Regression_eta: 0.0127 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7849 | Classification_binary: 2.4371 | Regression_cos_phi: 0.5950 | Regression_energy: 1.7739 | Regression_eta: 0.6245 | Regression_pt: 1.9373 | Regression_sin_phi: 0.4019 | LR: 5.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1229 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4726.2 | uss_MB: 3954.7 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:11:42,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:12:22,104] INFO : Step 24900/50000 rank0 | Train Loss: 7.9427 | Train Loss Unweighted: 2.7152 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4479 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1108 | Regression_eta: 0.0091 | Regression_pt: 0.1113 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8068 | Classification_binary: 2.4479 | Regression_cos_phi: 0.5387 | Regression_energy: 1.5323 | Regression_eta: 0.4457 | Regression_pt: 1.6750 | Regression_sin_phi: 0.4962 | LR: 5.53e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5266s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0247 | max_backward: 0.2034 | max_optimizer_step: 0.1271 | max_tensorboard_logging: 0.3350 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4724.1 | uss_MB: 3954.4 | child_rss_MB: 3989.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:12:22,316] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:02,068] INFO : Step 25000/50000 rank0 | Train Loss: 8.3151 | Train Loss Unweighted: 2.7759 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4929 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1193 | Regression_eta: 0.0100 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8282 | Classification_binary: 2.4929 | Regression_cos_phi: 0.6220 | Regression_energy: 1.6494 | Regression_eta: 0.4912 | Regression_pt: 1.7483 | Regression_sin_phi: 0.4831 | LR: 5.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6070s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0968 | max_backward: 0.2029 | max_optimizer_step: 0.1266 | max_tensorboard_logging: 0.3445 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4724.1 | uss_MB: 3954.6 | child_rss_MB: 3990.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:13:02,281] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:13:02,318] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-25000.pth +rank0 [2026-10-28 14:13:02,494] INFO : Running validation on rank0 +rank0 [2026-10-28 14:13:02,494] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:13:02,494] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:13:03,847] INFO : saved batch0_epoch25000.parquet +rank0 [2026-10-28 14:13:03,990] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:13:04,120] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:13:04,192] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:13:04,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:13:04,328] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:13:04,396] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:13:04,466] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:13:04,580] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:13:04,711] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:13:04,842] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:13:04,913] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:13:04,981] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:13:05,048] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:13:05,116] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:13:05,185] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:13:05,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:13:05,286] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:13:05,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:13:05,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:13:05,292] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:13:05,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:13:05,296] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:13:05,299] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:13:05,300] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:13:05,303] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:13:05,305] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:13:05,308] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:13:05,309] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:13:05,311] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:13:05,312] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:13:05,315] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:13:12,767] INFO : VALIDATION | Step=25000/50000 | Train Loss=8.3151 | Valid Loss=8.5592 | Train Loss Unweighted=2.7759 | Valid Loss Unweighted=2.7818 | Stale=0 | ETA=180.7m +rank0 [2026-10-28 14:13:12,777] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:12,852] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:13:12,855] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:13,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:14,983] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:15,611] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:16,290] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:16,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:17,134] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:13:17,137] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:17,193] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:13:17,196] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:17,297] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:19,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:19,873] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:20,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:21,153] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:21,281] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:13:21,284] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:13:21,349] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:13:21,352] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:13:21,443] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:13:23,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:13:23,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:13:24,546] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:13:25,216] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:13:25,364] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:13:25,371] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:25,626] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:13:26,073] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:27,497] INFO : Plotted jets +rank0 [2026-10-28 14:13:28,596] INFO : Plotted jet ratio +rank0 [2026-10-28 14:13:28,629] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:28,852] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:13:29,232] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:31,171] INFO : Plotted jets +rank0 [2026-10-28 14:13:32,283] INFO : Plotted jet ratio +rank0 [2026-10-28 14:13:32,316] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 14:13:32,520] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_25000/clic_edm_qq_pf +rank0 [2026-10-28 14:13:32,931] INFO : Plotted number of elements +rank0 [2026-10-28 14:13:34,314] INFO : Plotted jets +rank0 [2026-10-28 14:13:35,416] INFO : Plotted jet ratio +rank0 [2026-10-28 14:14:15,559] INFO : Step 25100/50000 rank0 | Train Loss: 9.0638 | Train Loss Unweighted: 2.8492 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5147 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1437 | Regression_eta: 0.0096 | Regression_pt: 0.1431 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8861 | Classification_binary: 2.5147 | Regression_cos_phi: 0.5793 | Regression_energy: 1.9874 | Regression_eta: 0.4702 | Regression_pt: 2.1534 | Regression_sin_phi: 0.4727 | LR: 5.47e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6327s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.2524 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3861 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4919.1 | uss_MB: 4149.4 | child_rss_MB: 3988.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:14:15,773] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:14:55,237] INFO : Step 25200/50000 rank0 | Train Loss: 8.3465 | Train Loss Unweighted: 2.8001 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5085 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1238 | Regression_eta: 0.0092 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8966 | Classification_binary: 2.5085 | Regression_cos_phi: 0.5010 | Regression_energy: 1.7126 | Regression_eta: 0.4505 | Regression_pt: 1.8341 | Regression_sin_phi: 0.4433 | LR: 5.44e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0654 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3088 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2812 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4919.1 | uss_MB: 4149.4 | child_rss_MB: 4011.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:14:55,451] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:15:35,122] INFO : Step 25300/50000 rank0 | Train Loss: 8.2620 | Train Loss Unweighted: 2.7743 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4785 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1272 | Regression_eta: 0.0080 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8791 | Classification_binary: 2.4785 | Regression_cos_phi: 0.4830 | Regression_energy: 1.7590 | Regression_eta: 0.3944 | Regression_pt: 1.8921 | Regression_sin_phi: 0.3759 | LR: 5.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5930s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0933 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.2921 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4798.4 | uss_MB: 4028.6 | child_rss_MB: 3988.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:15:35,337] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:14,887] INFO : Step 25400/50000 rank0 | Train Loss: 8.6310 | Train Loss Unweighted: 2.7921 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4896 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1275 | Regression_eta: 0.0100 | Regression_pt: 0.1266 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8310 | Classification_binary: 2.4896 | Regression_cos_phi: 0.6011 | Regression_energy: 1.7629 | Regression_eta: 0.4937 | Regression_pt: 1.9043 | Regression_sin_phi: 0.5485 | LR: 5.39e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6994s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.2824 | max_optimizer_step: 0.1299 | max_tensorboard_logging: 0.3879 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4798.4 | uss_MB: 4028.7 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:16:15,105] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:16:54,741] INFO : Step 25500/50000 rank0 | Train Loss: 8.1559 | Train Loss Unweighted: 2.7122 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4285 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1197 | Regression_eta: 0.0094 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8514 | Classification_binary: 2.4285 | Regression_cos_phi: 0.4926 | Regression_energy: 1.6554 | Regression_eta: 0.4597 | Regression_pt: 1.7805 | Regression_sin_phi: 0.4877 | LR: 5.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6721s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0034 | max_backward: 0.3076 | max_optimizer_step: 0.1304 | max_tensorboard_logging: 0.3741 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 4798.4 | uss_MB: 4028.9 | child_rss_MB: 3990.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:16:54,956] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:17:35,080] INFO : Step 25600/50000 rank0 | Train Loss: 8.3524 | Train Loss Unweighted: 2.7709 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4765 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1253 | Regression_eta: 0.0103 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8081 | Classification_binary: 2.4765 | Regression_cos_phi: 0.5107 | Regression_energy: 1.7326 | Regression_eta: 0.5071 | Regression_pt: 1.8644 | Regression_sin_phi: 0.4529 | LR: 5.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6607s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1836 | max_loss: 0.0037 | max_backward: 0.2029 | max_optimizer_step: 0.2203 | max_tensorboard_logging: 0.2827 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4798.4 | uss_MB: 4028.9 | child_rss_MB: 3992.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:17:35,308] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:18:15,795] INFO : Step 25700/50000 rank0 | Train Loss: 8.4961 | Train Loss Unweighted: 2.7574 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4381 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1382 | Regression_eta: 0.0084 | Regression_pt: 0.1395 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8782 | Classification_binary: 2.4381 | Regression_cos_phi: 0.3885 | Regression_energy: 1.9112 | Regression_eta: 0.4126 | Regression_pt: 2.0991 | Regression_sin_phi: 0.3684 | LR: 5.30e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6650s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1584 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2306 | max_tensorboard_logging: 0.2906 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4798.4 | uss_MB: 4029.0 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:18:16,019] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:18:56,186] INFO : Step 25800/50000 rank0 | Train Loss: 8.3694 | Train Loss Unweighted: 2.6767 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3769 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1270 | Regression_eta: 0.0086 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8370 | Classification_binary: 2.3769 | Regression_cos_phi: 0.6006 | Regression_energy: 1.7558 | Regression_eta: 0.4241 | Regression_pt: 1.9133 | Regression_sin_phi: 0.4617 | LR: 5.27e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6485s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2954 | max_optimizer_step: 0.1263 | max_tensorboard_logging: 0.3782 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4798.4 | uss_MB: 4029.0 | child_rss_MB: 3987.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:18:56,397] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:19:36,114] INFO : Step 25900/50000 rank0 | Train Loss: 9.1298 | Train Loss Unweighted: 2.9160 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.5905 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1389 | Regression_eta: 0.0101 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8386 | Classification_binary: 2.5905 | Regression_cos_phi: 0.6337 | Regression_energy: 1.9213 | Regression_eta: 0.4945 | Regression_pt: 2.0541 | Regression_sin_phi: 0.5971 | LR: 5.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6857s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3052 | max_optimizer_step: 0.1309 | max_tensorboard_logging: 0.3897 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 4798.4 | uss_MB: 4028.7 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:19:36,326] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:16,142] INFO : Step 26000/50000 rank0 | Train Loss: 9.1757 | Train Loss Unweighted: 2.8125 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.4804 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1423 | Regression_eta: 0.0115 | Regression_pt: 0.1377 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9118 | Classification_binary: 2.4804 | Regression_cos_phi: 0.6900 | Regression_energy: 1.9674 | Regression_eta: 0.5647 | Regression_pt: 2.0709 | Regression_sin_phi: 0.4903 | LR: 5.22e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6658s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0030 | max_backward: 0.3041 | max_optimizer_step: 0.1317 | max_tensorboard_logging: 0.3829 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 4798.4 | uss_MB: 4028.8 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:20:16,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:20:16,373] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-26000.pth +rank0 [2026-10-28 14:20:16,646] INFO : Running validation on rank0 +rank0 [2026-10-28 14:20:16,646] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:20:16,646] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:20:17,931] INFO : saved batch0_epoch26000.parquet +rank0 [2026-10-28 14:20:18,076] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:20:18,208] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:20:18,281] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:20:18,349] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:20:18,415] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:20:18,484] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:20:18,553] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:20:18,666] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:20:18,798] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:20:18,929] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:20:19,000] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:20:19,068] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:20:19,137] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:20:19,205] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:20:19,274] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:20:19,393] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:20:19,400] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:20:19,403] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:20:19,406] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:20:19,407] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:20:19,409] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:20:19,411] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:20:19,413] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:20:19,416] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:20:19,417] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:20:19,419] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:20:19,420] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:20:19,423] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:20:19,425] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:20:19,428] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:20:19,429] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:20:26,804] INFO : VALIDATION | Step=26000/50000 | Train Loss=9.1757 | Valid Loss=8.5511 | Train Loss Unweighted=2.8125 | Valid Loss Unweighted=2.7827 | Stale=0 | ETA=173.4m +rank0 [2026-10-28 14:20:26,815] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:26,897] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:20:26,900] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:27,042] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:29,043] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:29,712] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:30,407] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:31,076] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:31,211] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:31,215] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:31,271] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:20:31,274] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:31,377] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:33,304] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:34,007] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:34,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:35,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:35,417] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:35,420] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:20:35,474] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:20:35,477] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:20:35,585] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:20:37,667] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:20:38,333] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:20:38,913] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:20:39,549] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:20:39,698] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:20:39,705] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:39,961] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:20:40,408] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:41,830] INFO : Plotted jets +rank0 [2026-10-28 14:20:43,475] INFO : Plotted jet ratio +rank0 [2026-10-28 14:20:43,507] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:43,737] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:20:44,111] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:45,497] INFO : Plotted jets +rank0 [2026-10-28 14:20:46,559] INFO : Plotted jet ratio +rank0 [2026-10-28 14:20:46,592] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 14:20:46,799] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_26000/clic_edm_qq_pf +rank0 [2026-10-28 14:20:47,212] INFO : Plotted number of elements +rank0 [2026-10-28 14:20:48,593] INFO : Plotted jets +rank0 [2026-10-28 14:20:49,695] INFO : Plotted jet ratio +rank0 [2026-10-28 14:21:30,200] INFO : Step 26100/50000 rank0 | Train Loss: 8.9560 | Train Loss Unweighted: 2.7926 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4672 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1399 | Regression_eta: 0.0089 | Regression_pt: 0.1368 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8697 | Classification_binary: 2.4672 | Regression_cos_phi: 0.6913 | Regression_energy: 1.9341 | Regression_eta: 0.4382 | Regression_pt: 2.0581 | Regression_sin_phi: 0.4973 | LR: 5.19e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6871s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3074 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.3771 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5001.8 | uss_MB: 4231.7 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:21:30,414] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:10,826] INFO : Step 26200/50000 rank0 | Train Loss: 8.7222 | Train Loss Unweighted: 2.7342 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4306 | Regression_cos_phi: 0.0113 | Regression_energy: 0.1269 | Regression_eta: 0.0115 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7769 | Classification_binary: 2.4306 | Regression_cos_phi: 0.7515 | Regression_energy: 1.7547 | Regression_eta: 0.5648 | Regression_pt: 1.8911 | Regression_sin_phi: 0.5525 | LR: 5.16e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6461s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0323 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.3110 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5001.8 | uss_MB: 4231.8 | child_rss_MB: 4010.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:22:11,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:22:50,477] INFO : Step 26300/50000 rank0 | Train Loss: 8.0842 | Train Loss Unweighted: 2.6753 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3983 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1149 | Regression_eta: 0.0092 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8473 | Classification_binary: 2.3983 | Regression_cos_phi: 0.6224 | Regression_energy: 1.5885 | Regression_eta: 0.4516 | Regression_pt: 1.7421 | Regression_sin_phi: 0.4340 | LR: 5.13e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5870s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0548 | max_backward: 0.2037 | max_optimizer_step: 0.1293 | max_tensorboard_logging: 0.3418 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4872.2 | uss_MB: 4100.6 | child_rss_MB: 3985.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:22:50,690] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:23:30,295] INFO : Step 26400/50000 rank0 | Train Loss: 8.4589 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.4531 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1298 | Regression_eta: 0.0091 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7989 | Classification_binary: 2.4531 | Regression_cos_phi: 0.5453 | Regression_energy: 1.7944 | Regression_eta: 0.4453 | Regression_pt: 1.9162 | Regression_sin_phi: 0.5056 | LR: 5.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8219s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0206 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3953 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3956 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 4872.2 | uss_MB: 4103.7 | child_rss_MB: 3986.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 14:23:30,508] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:10,963] INFO : Step 26500/50000 rank0 | Train Loss: 8.4298 | Train Loss Unweighted: 2.6826 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3734 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1339 | Regression_eta: 0.0100 | Regression_pt: 0.1308 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8166 | Classification_binary: 2.3734 | Regression_cos_phi: 0.4833 | Regression_energy: 1.8512 | Regression_eta: 0.4927 | Regression_pt: 1.9675 | Regression_sin_phi: 0.4451 | LR: 5.08e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.6736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3053 | max_optimizer_step: 0.1300 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 4873.2 | uss_MB: 4101.9 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:24:11,177] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:24:50,909] INFO : Step 26600/50000 rank0 | Train Loss: 8.8323 | Train Loss Unweighted: 2.8044 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4920 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1322 | Regression_eta: 0.0128 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8143 | Classification_binary: 2.4920 | Regression_cos_phi: 0.6225 | Regression_energy: 1.8284 | Regression_eta: 0.6313 | Regression_pt: 1.9545 | Regression_sin_phi: 0.4892 | LR: 5.05e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6602s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0033 | max_backward: 0.3106 | max_optimizer_step: 0.1286 | max_tensorboard_logging: 0.3742 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 4873.2 | uss_MB: 4102.0 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:24:51,121] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:25:31,569] INFO : Step 26700/50000 rank0 | Train Loss: 8.9932 | Train Loss Unweighted: 2.7662 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4208 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1507 | Regression_eta: 0.0088 | Regression_pt: 0.1507 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8187 | Classification_binary: 2.4208 | Regression_cos_phi: 0.5504 | Regression_energy: 2.0846 | Regression_eta: 0.4315 | Regression_pt: 2.2673 | Regression_sin_phi: 0.4200 | LR: 5.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6988s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3113 | max_optimizer_step: 0.1313 | max_tensorboard_logging: 0.3867 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4873.2 | uss_MB: 4102.1 | child_rss_MB: 3988.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:25:31,783] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:26:11,559] INFO : Step 26800/50000 rank0 | Train Loss: 9.0570 | Train Loss Unweighted: 2.8307 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.5089 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1347 | Regression_eta: 0.0107 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9160 | Classification_binary: 2.5089 | Regression_cos_phi: 0.6399 | Regression_energy: 1.8634 | Regression_eta: 0.5249 | Regression_pt: 2.0282 | Regression_sin_phi: 0.5757 | LR: 4.99e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6625s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3091 | max_optimizer_step: 0.1295 | max_tensorboard_logging: 0.3769 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 4873.2 | uss_MB: 4102.1 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:26:11,774] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:26:51,686] INFO : Step 26900/50000 rank0 | Train Loss: 9.1927 | Train Loss Unweighted: 2.8242 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4926 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1421 | Regression_eta: 0.0125 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8390 | Classification_binary: 2.4926 | Regression_cos_phi: 0.7393 | Regression_energy: 1.9647 | Regression_eta: 0.6162 | Regression_pt: 2.0700 | Regression_sin_phi: 0.4707 | LR: 4.96e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.5838s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1845 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1292 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 4872.1 | uss_MB: 4100.7 | child_rss_MB: 3985.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:26:51,902] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:31,144] INFO : Step 27000/50000 rank0 | Train Loss: 9.5774 | Train Loss Unweighted: 2.9421 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.5997 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1458 | Regression_eta: 0.0152 | Regression_pt: 0.1407 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8953 | Classification_binary: 2.5997 | Regression_cos_phi: 0.6992 | Regression_energy: 2.0164 | Regression_eta: 0.7490 | Regression_pt: 2.1167 | Regression_sin_phi: 0.5011 | LR: 4.94e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5541s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0607 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 4873.1 | uss_MB: 4101.8 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:27:31,358] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 146.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:27:31,377] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-27000.pth +rank0 [2026-10-28 14:27:31,546] INFO : Running validation on rank0 +rank0 [2026-10-28 14:27:31,546] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:27:31,546] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:27:32,841] INFO : saved batch0_epoch27000.parquet +rank0 [2026-10-28 14:27:32,985] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:27:33,116] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:27:33,190] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:27:33,258] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:27:33,327] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:27:33,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:27:33,464] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:27:33,577] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:27:33,707] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:27:33,838] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:27:33,909] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:27:33,978] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:27:34,046] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:27:34,115] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:27:34,184] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:27:34,280] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:27:34,286] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:27:34,288] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:27:34,291] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:27:34,292] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:27:34,295] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:27:34,298] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:27:34,301] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:27:34,303] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:27:34,306] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:27:34,309] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:27:34,312] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:27:34,313] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:27:34,316] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:27:34,319] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:27:34,322] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:27:41,639] INFO : VALIDATION | Step=27000/50000 | Train Loss=9.5774 | Valid Loss=8.4622 | Train Loss Unweighted=2.9421 | Valid Loss Unweighted=2.7511 | Stale=0 | ETA=166.2m +rank0 [2026-10-28 14:27:41,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:41,709] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:27:41,712] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:41,998] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:43,907] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:44,538] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:45,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:45,800] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:45,937] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:27:45,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:46,015] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:27:46,018] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:46,102] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:47,988] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:48,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:49,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:49,851] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:49,985] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:27:49,989] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:27:50,074] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:27:50,076] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:27:50,152] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:27:51,986] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:27:52,638] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:27:53,326] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:27:54,000] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:27:54,138] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:27:54,146] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:54,400] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:27:54,848] INFO : Plotted number of elements +rank0 [2026-10-28 14:27:56,271] INFO : Plotted jets +rank0 [2026-10-28 14:27:57,894] INFO : Plotted jet ratio +rank0 [2026-10-28 14:27:57,927] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:58,156] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:27:58,529] INFO : Plotted number of elements +rank0 [2026-10-28 14:27:59,908] INFO : Plotted jets +rank0 [2026-10-28 14:28:00,970] INFO : Plotted jet ratio +rank0 [2026-10-28 14:28:01,003] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 14:28:01,207] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_27000/clic_edm_qq_pf +rank0 [2026-10-28 14:28:01,630] INFO : Plotted number of elements +rank0 [2026-10-28 14:28:03,016] INFO : Plotted jets +rank0 [2026-10-28 14:28:04,123] INFO : Plotted jet ratio +rank0 [2026-10-28 14:28:43,777] INFO : Step 27100/50000 rank0 | Train Loss: 8.3222 | Train Loss Unweighted: 2.6726 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3747 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1265 | Regression_eta: 0.0093 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8360 | Classification_binary: 2.3747 | Regression_cos_phi: 0.5499 | Regression_energy: 1.7490 | Regression_eta: 0.4553 | Regression_pt: 1.8922 | Regression_sin_phi: 0.4652 | LR: 4.91e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7290s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1044 | max_loss: 0.0037 | max_backward: 0.3473 | max_optimizer_step: 0.1314 | max_tensorboard_logging: 0.3521 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5066.2 | uss_MB: 4301.1 | child_rss_MB: 3984.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 14:28:43,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:29:23,211] INFO : Step 27200/50000 rank0 | Train Loss: 8.2692 | Train Loss Unweighted: 2.6978 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4050 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1232 | Regression_eta: 0.0085 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8266 | Classification_binary: 2.4050 | Regression_cos_phi: 0.5257 | Regression_energy: 1.7040 | Regression_eta: 0.4176 | Regression_pt: 1.8712 | Regression_sin_phi: 0.5190 | LR: 4.88e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1830 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1331 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5066.2 | uss_MB: 4299.0 | child_rss_MB: 3982.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:29:23,428] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:03,187] INFO : Step 27300/50000 rank0 | Train Loss: 7.9742 | Train Loss Unweighted: 2.6823 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4038 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1192 | Regression_eta: 0.0099 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7716 | Classification_binary: 2.4038 | Regression_cos_phi: 0.4863 | Regression_energy: 1.6482 | Regression_eta: 0.4843 | Regression_pt: 1.7466 | Regression_sin_phi: 0.4334 | LR: 4.85e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.5992s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3020 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 4941.5 | uss_MB: 4174.1 | child_rss_MB: 3986.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:03,403] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:30:43,290] INFO : Step 27400/50000 rank0 | Train Loss: 8.3295 | Train Loss Unweighted: 2.7295 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.4511 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1143 | Regression_eta: 0.0109 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8903 | Classification_binary: 2.4511 | Regression_cos_phi: 0.6428 | Regression_energy: 1.5804 | Regression_eta: 0.5361 | Regression_pt: 1.6998 | Regression_sin_phi: 0.5289 | LR: 4.82e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6295s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1041 | max_loss: 0.0034 | max_backward: 0.2037 | max_optimizer_step: 0.2289 | max_tensorboard_logging: 0.2990 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 4942.5 | uss_MB: 4175.2 | child_rss_MB: 3984.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:30:43,506] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:31:23,275] INFO : Step 27500/50000 rank0 | Train Loss: 8.0114 | Train Loss Unweighted: 2.6226 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.3414 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1191 | Regression_eta: 0.0079 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8256 | Classification_binary: 2.3414 | Regression_cos_phi: 0.4837 | Regression_energy: 1.6472 | Regression_eta: 0.3895 | Regression_pt: 1.7675 | Regression_sin_phi: 0.5566 | LR: 4.80e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1190 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2333 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4942.5 | uss_MB: 4175.4 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:31:23,489] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:03,015] INFO : Step 27600/50000 rank0 | Train Loss: 8.8212 | Train Loss Unweighted: 2.7518 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.4465 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1278 | Regression_eta: 0.0134 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8434 | Classification_binary: 2.4465 | Regression_cos_phi: 0.7207 | Regression_energy: 1.7676 | Regression_eta: 0.6600 | Regression_pt: 1.8677 | Regression_sin_phi: 0.5153 | LR: 4.77e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5487s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0157 | max_backward: 0.2037 | max_optimizer_step: 0.1298 | max_tensorboard_logging: 0.3247 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4942.5 | uss_MB: 4175.6 | child_rss_MB: 3981.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 14:32:03,232] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:32:43,166] INFO : Step 27700/50000 rank0 | Train Loss: 7.7818 | Train Loss Unweighted: 2.6418 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3699 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1141 | Regression_eta: 0.0083 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8339 | Classification_binary: 2.3699 | Regression_cos_phi: 0.4699 | Regression_energy: 1.5780 | Regression_eta: 0.4066 | Regression_pt: 1.7442 | Regression_sin_phi: 0.3793 | LR: 4.74e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0050 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3056 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 4942.5 | uss_MB: 4175.7 | child_rss_MB: 4003.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:32:43,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:33:22,842] INFO : Step 27800/50000 rank0 | Train Loss: 8.3880 | Train Loss Unweighted: 2.7172 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4061 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1347 | Regression_eta: 0.0081 | Regression_pt: 0.1348 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7885 | Classification_binary: 2.4061 | Regression_cos_phi: 0.4675 | Regression_energy: 1.8621 | Regression_eta: 0.3974 | Regression_pt: 2.0287 | Regression_sin_phi: 0.4378 | LR: 4.71e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5808s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1851 | max_loss: 0.0034 | max_backward: 0.2033 | max_optimizer_step: 0.1288 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 4942.5 | uss_MB: 4175.8 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:33:23,059] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:03,071] INFO : Step 27900/50000 rank0 | Train Loss: 8.0460 | Train Loss Unweighted: 2.6968 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.4187 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1170 | Regression_eta: 0.0100 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7294 | Classification_binary: 2.4187 | Regression_cos_phi: 0.5654 | Regression_energy: 1.6181 | Regression_eta: 0.4890 | Regression_pt: 1.7626 | Regression_sin_phi: 0.4629 | LR: 4.68e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6406s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0353 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3103 | max_optimizer_step: 0.1307 | max_tensorboard_logging: 0.2915 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 4941.1 | uss_MB: 4174.1 | child_rss_MB: 4007.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:03,283] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:42,693] INFO : Step 28000/50000 rank0 | Train Loss: 8.4309 | Train Loss Unweighted: 2.7268 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.4223 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1295 | Regression_eta: 0.0094 | Regression_pt: 0.1299 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8464 | Classification_binary: 2.4223 | Regression_cos_phi: 0.4994 | Regression_energy: 1.7907 | Regression_eta: 0.4643 | Regression_pt: 1.9546 | Regression_sin_phi: 0.4532 | LR: 4.66e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6453s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0034 | max_backward: 0.2511 | max_optimizer_step: 0.1287 | max_tensorboard_logging: 0.3954 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 4941.1 | uss_MB: 4175.3 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:34:42,909] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:34:42,970] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-28000.pth +rank0 [2026-10-28 14:34:43,143] INFO : Running validation on rank0 +rank0 [2026-10-28 14:34:43,143] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:34:43,143] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:34:44,444] INFO : saved batch0_epoch28000.parquet +rank0 [2026-10-28 14:34:44,590] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:34:44,719] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:34:44,792] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:34:44,859] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:34:44,927] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:34:44,997] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:34:45,065] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:34:45,181] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:34:45,312] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:34:45,443] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:34:45,514] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:34:45,582] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:34:45,649] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:34:45,717] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:34:45,787] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:34:45,885] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:34:45,892] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:34:45,894] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:34:45,895] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:34:45,898] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:34:45,900] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:34:45,902] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:34:45,904] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:34:45,905] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:34:45,908] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:34:45,909] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:34:45,912] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:34:45,913] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:34:45,915] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:34:45,916] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:34:45,919] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:34:53,360] INFO : VALIDATION | Step=28000/50000 | Train Loss=8.4309 | Valid Loss=8.4196 | Train Loss Unweighted=2.7268 | Valid Loss Unweighted=2.7364 | Stale=0 | ETA=159.0m +rank0 [2026-10-28 14:34:53,371] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:53,432] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:34:53,436] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:34:53,603] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:34:55,556] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:34:56,210] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:34:56,929] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:34:57,642] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:34:57,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:34:57,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:34:57,837] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:34:57,840] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:34:57,945] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:34:59,860] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:35:00,467] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:35:01,157] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:35:01,753] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:35:01,889] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:35:01,893] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:35:01,946] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:35:01,949] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:35:02,054] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:35:03,815] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:35:04,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:35:05,155] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:35:05,829] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:35:05,975] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:35:05,983] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:35:06,237] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:35:06,686] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:08,112] INFO : Plotted jets +rank0 [2026-10-28 14:35:09,739] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:09,772] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:35:09,998] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:35:10,371] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:11,745] INFO : Plotted jets +rank0 [2026-10-28 14:35:12,812] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:12,847] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 14:35:13,052] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_28000/clic_edm_qq_pf +rank0 [2026-10-28 14:35:13,463] INFO : Plotted number of elements +rank0 [2026-10-28 14:35:14,844] INFO : Plotted jets +rank0 [2026-10-28 14:35:15,940] INFO : Plotted jet ratio +rank0 [2026-10-28 14:35:56,473] INFO : Step 28100/50000 rank0 | Train Loss: 7.9455 | Train Loss Unweighted: 2.7135 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.4332 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1205 | Regression_eta: 0.0077 | Regression_pt: 0.1191 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7746 | Classification_binary: 2.4332 | Regression_cos_phi: 0.4995 | Regression_energy: 1.6666 | Regression_eta: 0.3792 | Regression_pt: 1.7914 | Regression_sin_phi: 0.4010 | LR: 4.63e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7237s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1858 | max_loss: 0.0034 | max_backward: 0.2035 | max_optimizer_step: 0.2355 | max_tensorboard_logging: 0.2886 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5133.5 | uss_MB: 4367.7 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:35:56,689] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:36:36,546] INFO : Step 28200/50000 rank0 | Train Loss: 8.6838 | Train Loss Unweighted: 2.8756 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.5627 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1352 | Regression_eta: 0.0083 | Regression_pt: 0.1327 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8806 | Classification_binary: 2.5627 | Regression_cos_phi: 0.5001 | Regression_energy: 1.8698 | Regression_eta: 0.4061 | Regression_pt: 1.9964 | Regression_sin_phi: 0.4680 | LR: 4.60e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0845 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3108 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2795 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5133.5 | uss_MB: 4367.7 | child_rss_MB: 4007.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:36:36,759] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:17,137] INFO : Step 28300/50000 rank0 | Train Loss: 8.1447 | Train Loss Unweighted: 2.8411 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5674 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1168 | Regression_eta: 0.0093 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8325 | Classification_binary: 2.5674 | Regression_cos_phi: 0.5565 | Regression_energy: 1.6151 | Regression_eta: 0.4590 | Regression_pt: 1.6806 | Regression_sin_phi: 0.4335 | LR: 4.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6289s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0805 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.3056 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5014.0 | uss_MB: 4247.1 | child_rss_MB: 3984.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:37:17,355] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:37:57,283] INFO : Step 28400/50000 rank0 | Train Loss: 8.8517 | Train Loss Unweighted: 2.7778 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.4528 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1410 | Regression_eta: 0.0107 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8893 | Classification_binary: 2.4528 | Regression_cos_phi: 0.5220 | Regression_energy: 1.9497 | Regression_eta: 0.5246 | Regression_pt: 2.0476 | Regression_sin_phi: 0.4657 | LR: 4.55e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6771s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0628 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3107 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3042 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5016.0 | uss_MB: 4248.2 | child_rss_MB: 4013.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:37:57,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:38:37,084] INFO : Step 28500/50000 rank0 | Train Loss: 8.5647 | Train Loss Unweighted: 2.7590 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.4571 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1277 | Regression_eta: 0.0084 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8217 | Classification_binary: 2.4571 | Regression_cos_phi: 0.6435 | Regression_energy: 1.7656 | Regression_eta: 0.4126 | Regression_pt: 1.9063 | Regression_sin_phi: 0.5578 | LR: 4.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6478s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1389 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.2319 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5016.0 | uss_MB: 4248.4 | child_rss_MB: 3984.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:38:37,312] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:17,519] INFO : Step 28600/50000 rank0 | Train Loss: 8.1791 | Train Loss Unweighted: 2.7552 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4752 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1186 | Regression_eta: 0.0094 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7609 | Classification_binary: 2.4752 | Regression_cos_phi: 0.5793 | Regression_energy: 1.6402 | Regression_eta: 0.4599 | Regression_pt: 1.7473 | Regression_sin_phi: 0.5164 | LR: 4.49e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0778 | max_loss: 0.0443 | max_backward: 0.2032 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3227 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:39:17,735] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:39:57,500] INFO : Step 28700/50000 rank0 | Train Loss: 8.8816 | Train Loss Unweighted: 2.7869 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4682 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1363 | Regression_eta: 0.0114 | Regression_pt: 0.1322 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8648 | Classification_binary: 2.4682 | Regression_cos_phi: 0.5636 | Regression_energy: 1.8846 | Regression_eta: 0.5625 | Regression_pt: 1.9893 | Regression_sin_phi: 0.5485 | LR: 4.46e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6124s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0255 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.3114 | max_optimizer_step: 0.1323 | max_tensorboard_logging: 0.2785 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 4006.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:39:57,716] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:40:37,751] INFO : Step 28800/50000 rank0 | Train Loss: 8.0992 | Train Loss Unweighted: 2.5726 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.2823 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1228 | Regression_eta: 0.0097 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7673 | Classification_binary: 2.2823 | Regression_cos_phi: 0.5703 | Regression_energy: 1.6982 | Regression_eta: 0.4764 | Regression_pt: 1.8494 | Regression_sin_phi: 0.4553 | LR: 4.44e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7141s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.3114 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5016.0 | uss_MB: 4248.5 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:40:37,964] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:17,229] INFO : Step 28900/50000 rank0 | Train Loss: 8.1676 | Train Loss Unweighted: 2.6658 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3724 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1264 | Regression_eta: 0.0099 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7497 | Classification_binary: 2.3724 | Regression_cos_phi: 0.4947 | Regression_energy: 1.7478 | Regression_eta: 0.4882 | Regression_pt: 1.8642 | Regression_sin_phi: 0.4506 | LR: 4.41e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1289 | max_tensorboard_logging: 0.2933 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5015.8 | uss_MB: 4247.0 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:41:17,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:57,128] INFO : Step 29000/50000 rank0 | Train Loss: 8.1378 | Train Loss Unweighted: 2.6649 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3877 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1162 | Regression_eta: 0.0107 | Regression_pt: 0.1131 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7995 | Classification_binary: 2.3877 | Regression_cos_phi: 0.5456 | Regression_energy: 1.6063 | Regression_eta: 0.5244 | Regression_pt: 1.7019 | Regression_sin_phi: 0.5724 | LR: 4.38e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.5862s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0724 | max_backward: 0.2028 | max_optimizer_step: 0.1302 | max_tensorboard_logging: 0.3270 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5016.8 | uss_MB: 4248.1 | child_rss_MB: 3987.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:41:57,340] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:41:57,382] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-29000.pth +rank0 [2026-10-28 14:41:57,558] INFO : Running validation on rank0 +rank0 [2026-10-28 14:41:57,558] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:41:57,558] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:41:58,883] INFO : saved batch0_epoch29000.parquet +rank0 [2026-10-28 14:41:59,028] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:41:59,157] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:41:59,230] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:41:59,298] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:41:59,365] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:41:59,434] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:41:59,502] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:41:59,615] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:41:59,745] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:41:59,875] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:41:59,946] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:42:00,013] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:42:00,081] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:42:00,150] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:42:00,218] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:42:00,316] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:42:00,322] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:42:00,325] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:42:00,328] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:42:00,329] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:42:00,331] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:42:00,334] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:42:00,335] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:42:00,337] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:42:00,340] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:42:00,341] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:42:00,343] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:42:00,344] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:42:00,346] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:42:00,347] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:42:00,350] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:42:07,770] INFO : VALIDATION | Step=29000/50000 | Train Loss=8.1378 | Valid Loss=8.4468 | Train Loss Unweighted=2.6649 | Valid Loss Unweighted=2.7471 | Stale=1 | ETA=151.8m +rank0 [2026-10-28 14:42:07,781] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:07,843] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:42:07,847] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:08,014] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:09,928] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:10,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:11,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:12,063] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:12,194] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:12,198] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:12,253] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:42:12,256] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:12,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:14,273] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:14,881] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:15,565] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:16,166] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:16,352] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:16,355] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:42:16,409] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:42:16,412] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:42:16,517] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:42:18,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:42:18,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:42:19,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:42:20,312] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:42:20,476] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:42:20,483] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:20,741] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:42:21,190] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:22,610] INFO : Plotted jets +rank0 [2026-10-28 14:42:24,229] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:24,262] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:24,488] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:42:24,861] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:26,236] INFO : Plotted jets +rank0 [2026-10-28 14:42:27,295] INFO : Plotted jet ratio +rank0 [2026-10-28 14:42:27,327] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 14:42:27,531] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_29000/clic_edm_qq_pf +rank0 [2026-10-28 14:42:27,942] INFO : Plotted number of elements +rank0 [2026-10-28 14:42:29,321] INFO : Plotted jets +rank0 [2026-10-28 14:42:30,425] INFO : Plotted jet ratio +rank0 [2026-10-28 14:43:10,094] INFO : Step 29100/50000 rank0 | Train Loss: 8.0370 | Train Loss Unweighted: 2.6942 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.4143 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1194 | Regression_eta: 0.0090 | Regression_pt: 0.1173 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7500 | Classification_binary: 2.4143 | Regression_cos_phi: 0.5858 | Regression_energy: 1.6510 | Regression_eta: 0.4424 | Regression_pt: 1.7641 | Regression_sin_phi: 0.4294 | LR: 4.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.5940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0575 | max_backward: 0.2030 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3150 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5209.6 | uss_MB: 4441.9 | child_rss_MB: 3984.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 14:43:10,306] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:43:49,655] INFO : Step 29200/50000 rank0 | Train Loss: 8.6408 | Train Loss Unweighted: 2.7512 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4435 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1316 | Regression_eta: 0.0102 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8166 | Classification_binary: 2.4435 | Regression_cos_phi: 0.6314 | Regression_energy: 1.8195 | Regression_eta: 0.4991 | Regression_pt: 1.9296 | Regression_sin_phi: 0.5011 | LR: 4.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5963s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.2381 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5209.6 | uss_MB: 4441.9 | child_rss_MB: 3985.1 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:43:49,881] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:44:30,789] INFO : Step 29300/50000 rank0 | Train Loss: 8.7085 | Train Loss Unweighted: 2.8323 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.5252 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1285 | Regression_eta: 0.0096 | Regression_pt: 0.1305 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8329 | Classification_binary: 2.5252 | Regression_cos_phi: 0.5330 | Regression_energy: 1.7776 | Regression_eta: 0.4728 | Regression_pt: 1.9635 | Regression_sin_phi: 0.6036 | LR: 4.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6003s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0516 | max_backward: 0.2030 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.3235 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4321.2 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:44:31,001] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:10,803] INFO : Step 29400/50000 rank0 | Train Loss: 8.5965 | Train Loss Unweighted: 2.7093 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4049 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1288 | Regression_eta: 0.0112 | Regression_pt: 0.1265 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 2.4049 | Regression_cos_phi: 0.6475 | Regression_energy: 1.7806 | Regression_eta: 0.5525 | Regression_pt: 1.9038 | Regression_sin_phi: 0.4811 | LR: 4.27e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5439s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0187 | max_backward: 0.2035 | max_optimizer_step: 0.1333 | max_tensorboard_logging: 0.3153 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4322.5 | child_rss_MB: 3987.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:45:11,016] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 147.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:45:50,347] INFO : Step 29500/50000 rank0 | Train Loss: 8.5762 | Train Loss Unweighted: 2.6817 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.3742 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1318 | Regression_eta: 0.0100 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8382 | Classification_binary: 2.3742 | Regression_cos_phi: 0.6068 | Regression_energy: 1.8231 | Regression_eta: 0.4925 | Regression_pt: 1.9150 | Regression_sin_phi: 0.5264 | LR: 4.24e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6919s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1606 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.2311 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 3984.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:45:50,563] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:46:29,657] INFO : Step 29600/50000 rank0 | Train Loss: 8.7316 | Train Loss Unweighted: 2.7767 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4712 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1294 | Regression_eta: 0.0097 | Regression_pt: 0.1260 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8534 | Classification_binary: 2.4712 | Regression_cos_phi: 0.6981 | Regression_energy: 1.7887 | Regression_eta: 0.4775 | Regression_pt: 1.8950 | Regression_sin_phi: 0.5477 | LR: 4.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0814 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2344 | max_tensorboard_logging: 0.2979 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 3991.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:46:29,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:09,745] INFO : Step 29700/50000 rank0 | Train Loss: 8.3128 | Train Loss Unweighted: 2.6743 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.3783 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1256 | Regression_eta: 0.0099 | Regression_pt: 0.1244 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8165 | Classification_binary: 2.3783 | Regression_cos_phi: 0.5460 | Regression_energy: 1.7366 | Regression_eta: 0.4875 | Regression_pt: 1.8720 | Regression_sin_phi: 0.4758 | LR: 4.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6223s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0132 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1315 | max_tensorboard_logging: 0.2980 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5089.6 | uss_MB: 4322.7 | child_rss_MB: 4015.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:09,958] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:47:50,576] INFO : Step 29800/50000 rank0 | Train Loss: 8.1765 | Train Loss Unweighted: 2.7690 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4840 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1225 | Regression_eta: 0.0098 | Regression_pt: 0.1187 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7914 | Classification_binary: 2.4840 | Regression_cos_phi: 0.5314 | Regression_energy: 1.6946 | Regression_eta: 0.4814 | Regression_pt: 1.7851 | Regression_sin_phi: 0.4087 | LR: 4.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6396s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0644 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.3074 | max_optimizer_step: 0.1327 | max_tensorboard_logging: 0.2796 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 5089.6 | uss_MB: 4322.8 | child_rss_MB: 3988.3 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:47:50,792] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:48:30,607] INFO : Step 29900/50000 rank0 | Train Loss: 8.0130 | Train Loss Unweighted: 2.7386 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.4602 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1187 | Regression_eta: 0.0089 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.4602 | Regression_cos_phi: 0.5121 | Regression_energy: 1.6420 | Regression_eta: 0.4363 | Regression_pt: 1.7674 | Regression_sin_phi: 0.4617 | LR: 4.14e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6982s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0932 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3061 | max_optimizer_step: 0.1303 | max_tensorboard_logging: 0.2987 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5088.6 | uss_MB: 4321.1 | child_rss_MB: 4010.0 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:48:30,821] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:10,833] INFO : Step 30000/50000 rank0 | Train Loss: 7.9866 | Train Loss Unweighted: 2.6757 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3953 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1209 | Regression_eta: 0.0085 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8436 | Classification_binary: 2.3953 | Regression_cos_phi: 0.4803 | Regression_energy: 1.6718 | Regression_eta: 0.4195 | Regression_pt: 1.7476 | Regression_sin_phi: 0.4286 | LR: 4.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6891s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3066 | max_optimizer_step: 0.1344 | max_tensorboard_logging: 0.3784 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5089.6 | uss_MB: 4322.4 | child_rss_MB: 3985.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:49:11,046] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:49:11,064] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-30000.pth +rank0 [2026-10-28 14:49:11,357] INFO : Running validation on rank0 +rank0 [2026-10-28 14:49:11,357] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:49:11,357] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:49:12,664] INFO : saved batch0_epoch30000.parquet +rank0 [2026-10-28 14:49:12,807] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:49:12,937] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:49:13,010] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:49:13,078] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:49:13,145] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:49:13,213] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:49:13,282] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:49:13,395] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:49:13,525] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:49:13,655] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:49:13,726] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:49:13,795] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:49:13,863] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:49:13,932] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:49:14,002] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:49:14,100] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:49:14,107] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:49:14,109] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:49:14,110] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:49:14,113] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:49:14,114] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:49:14,117] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:49:14,119] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:49:14,120] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:49:14,123] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:49:14,124] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:49:14,126] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:49:14,129] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:49:14,129] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:49:14,132] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:49:14,134] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:49:21,502] INFO : VALIDATION | Step=30000/50000 | Train Loss=7.9866 | Valid Loss=8.3367 | Train Loss Unweighted=2.6757 | Valid Loss Unweighted=2.7139 | Stale=0 | ETA=144.5m +rank0 [2026-10-28 14:49:21,509] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:21,572] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:49:21,575] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:21,650] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:23,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:24,307] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:24,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:25,664] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:25,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:25,816] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:25,872] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:49:25,875] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:25,986] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:27,942] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:28,594] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:29,206] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:29,856] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:30,021] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:30,024] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:49:30,079] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:49:30,082] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:49:30,187] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:49:32,128] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:49:32,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:49:33,357] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:49:34,031] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:49:34,179] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:49:34,187] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:34,444] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:49:34,892] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:36,320] INFO : Plotted jets +rank0 [2026-10-28 14:49:37,424] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:37,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:37,677] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:49:38,045] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:39,968] INFO : Plotted jets +rank0 [2026-10-28 14:49:41,041] INFO : Plotted jet ratio +rank0 [2026-10-28 14:49:41,073] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 14:49:41,277] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_30000/clic_edm_qq_pf +rank0 [2026-10-28 14:49:41,689] INFO : Plotted number of elements +rank0 [2026-10-28 14:49:43,073] INFO : Plotted jets +rank0 [2026-10-28 14:49:44,175] INFO : Plotted jet ratio +rank0 [2026-10-28 14:50:23,997] INFO : Step 30100/50000 rank0 | Train Loss: 8.0665 | Train Loss Unweighted: 2.6752 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3797 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1286 | Regression_eta: 0.0079 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7370 | Classification_binary: 2.3797 | Regression_cos_phi: 0.4569 | Regression_energy: 1.7789 | Regression_eta: 0.3883 | Regression_pt: 1.9149 | Regression_sin_phi: 0.4109 | LR: 4.08e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6387s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1354 | max_tensorboard_logging: 0.2916 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5285.6 | uss_MB: 4521.1 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:50:24,216] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:04,584] INFO : Step 30200/50000 rank0 | Train Loss: 8.1842 | Train Loss Unweighted: 2.7650 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.4895 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1134 | Regression_eta: 0.0105 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8585 | Classification_binary: 2.4895 | Regression_cos_phi: 0.5345 | Regression_energy: 1.5686 | Regression_eta: 0.5154 | Regression_pt: 1.7216 | Regression_sin_phi: 0.4961 | LR: 4.06e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6242s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0783 | max_backward: 0.2036 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3229 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5285.6 | uss_MB: 4521.2 | child_rss_MB: 3987.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:04,797] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:51:44,963] INFO : Step 30300/50000 rank0 | Train Loss: 8.9950 | Train Loss Unweighted: 2.8466 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.5274 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1365 | Regression_eta: 0.0101 | Regression_pt: 0.1319 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8733 | Classification_binary: 2.5274 | Regression_cos_phi: 0.6648 | Regression_energy: 1.8876 | Regression_eta: 0.4943 | Regression_pt: 1.9843 | Regression_sin_phi: 0.5633 | LR: 4.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6933s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0435 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3047 | max_optimizer_step: 0.1384 | max_tensorboard_logging: 0.2936 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5285.6 | uss_MB: 4521.2 | child_rss_MB: 4012.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:51:45,178] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:52:25,255] INFO : Step 30400/50000 rank0 | Train Loss: 8.3280 | Train Loss Unweighted: 2.7375 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.4297 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1344 | Regression_eta: 0.0073 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8246 | Classification_binary: 2.4297 | Regression_cos_phi: 0.4167 | Regression_energy: 1.8589 | Regression_eta: 0.3607 | Regression_pt: 1.9914 | Regression_sin_phi: 0.4461 | LR: 4.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6555s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2239 | max_optimizer_step: 0.1350 | max_tensorboard_logging: 0.3862 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5166.6 | uss_MB: 4399.5 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:52:25,469] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:05,217] INFO : Step 30500/50000 rank0 | Train Loss: 8.2259 | Train Loss Unweighted: 2.7119 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4219 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1242 | Regression_eta: 0.0088 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8187 | Classification_binary: 2.4219 | Regression_cos_phi: 0.5728 | Regression_energy: 1.7178 | Regression_eta: 0.4328 | Regression_pt: 1.8229 | Regression_sin_phi: 0.4390 | LR: 3.98e-04 | DataLoad Time: 0.0029s | Model Forward Time: 1.6970s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0035 | max_backward: 0.3097 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.3773 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5167.6 | uss_MB: 4399.7 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:53:05,429] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:53:45,509] INFO : Step 30600/50000 rank0 | Train Loss: 8.4804 | Train Loss Unweighted: 2.8380 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.5315 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1322 | Regression_eta: 0.0090 | Regression_pt: 0.1311 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8260 | Classification_binary: 2.5315 | Regression_cos_phi: 0.3949 | Regression_energy: 1.8284 | Regression_eta: 0.4398 | Regression_pt: 1.9722 | Regression_sin_phi: 0.4875 | LR: 3.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7204s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1797 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2372 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5167.6 | uss_MB: 4399.9 | child_rss_MB: 3981.7 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 14:53:45,723] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:54:25,640] INFO : Step 30700/50000 rank0 | Train Loss: 8.3313 | Train Loss Unweighted: 2.7364 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4408 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1266 | Regression_eta: 0.0112 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7721 | Classification_binary: 2.4408 | Regression_cos_phi: 0.5403 | Regression_energy: 1.7504 | Regression_eta: 0.5521 | Regression_pt: 1.8679 | Regression_sin_phi: 0.4078 | LR: 3.92e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6736s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0384 | max_forward: 0.0781 | max_loss: 0.0035 | max_backward: 0.3102 | max_optimizer_step: 0.1348 | max_tensorboard_logging: 0.2972 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5167.6 | uss_MB: 4400.0 | child_rss_MB: 4010.2 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:54:25,855] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:06,039] INFO : Step 30800/50000 rank0 | Train Loss: 8.3344 | Train Loss Unweighted: 2.6122 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3147 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1261 | Regression_eta: 0.0099 | Regression_pt: 0.1245 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7661 | Classification_binary: 2.3147 | Regression_cos_phi: 0.5546 | Regression_energy: 1.7431 | Regression_eta: 0.4853 | Regression_pt: 1.8736 | Regression_sin_phi: 0.5969 | LR: 3.90e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6229s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0885 | max_backward: 0.2031 | max_optimizer_step: 0.1322 | max_tensorboard_logging: 0.3259 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5167.6 | uss_MB: 4400.1 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:55:06,254] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:55:46,639] INFO : Step 30900/50000 rank0 | Train Loss: 8.2921 | Train Loss Unweighted: 2.7752 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.4775 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1283 | Regression_eta: 0.0078 | Regression_pt: 0.1274 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7970 | Classification_binary: 2.4775 | Regression_cos_phi: 0.4953 | Regression_energy: 1.7743 | Regression_eta: 0.3812 | Regression_pt: 1.9164 | Regression_sin_phi: 0.4504 | LR: 3.87e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7043s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3094 | max_optimizer_step: 0.1334 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5167.6 | uss_MB: 4400.1 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 14:55:46,851] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:56:26,824] INFO : Step 31000/50000 rank0 | Train Loss: 7.9653 | Train Loss Unweighted: 2.6458 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3765 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1117 | Regression_eta: 0.0084 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7758 | Classification_binary: 2.3765 | Regression_cos_phi: 0.6365 | Regression_energy: 1.5451 | Regression_eta: 0.4142 | Regression_pt: 1.6802 | Regression_sin_phi: 0.5370 | LR: 3.84e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6019s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1783 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.1310 | max_tensorboard_logging: 0.2862 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5168.4 | uss_MB: 4399.7 | child_rss_MB: 3984.4 | allocated_MB: 291.4 | reserved_MB: 3840.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 14:56:27,041] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:56:27,060] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-31000.pth +rank0 [2026-10-28 14:56:27,232] INFO : Running validation on rank0 +rank0 [2026-10-28 14:56:27,232] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 14:56:27,233] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 14:56:28,531] INFO : saved batch0_epoch31000.parquet +rank0 [2026-10-28 14:56:28,676] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 14:56:28,805] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 14:56:28,879] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 14:56:28,946] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 14:56:29,013] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 14:56:29,082] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 14:56:29,151] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 14:56:29,263] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 14:56:29,394] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 14:56:29,525] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 14:56:29,595] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 14:56:29,664] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 14:56:29,732] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 14:56:29,800] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 14:56:29,869] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 14:56:29,966] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 14:56:29,971] INFO : plotted pt_target histogram +rank0 [2026-10-28 14:56:29,974] INFO : plotted pt_pred histogram +rank0 [2026-10-28 14:56:29,975] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 14:56:29,978] INFO : plotted eta_target histogram +rank0 [2026-10-28 14:56:29,980] INFO : plotted eta_pred histogram +rank0 [2026-10-28 14:56:29,982] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 14:56:29,984] INFO : plotted sphi_target histogram +rank0 [2026-10-28 14:56:29,985] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 14:56:29,988] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 14:56:29,989] INFO : plotted cphi_target histogram +rank0 [2026-10-28 14:56:29,991] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 14:56:29,993] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 14:56:29,995] INFO : plotted energy_target histogram +rank0 [2026-10-28 14:56:29,996] INFO : plotted energy_pred histogram +rank0 [2026-10-28 14:56:29,998] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 14:56:37,286] INFO : VALIDATION | Step=31000/50000 | Train Loss=7.9653 | Valid Loss=8.3178 | Train Loss Unweighted=2.6458 | Valid Loss Unweighted=2.7033 | Stale=0 | ETA=137.3m +rank0 [2026-10-28 14:56:37,297] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:37,360] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 14:56:37,363] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:37,538] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:39,509] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:40,209] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:40,934] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:41,636] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:41,776] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:56:41,779] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:41,835] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 14:56:41,838] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:41,942] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:43,872] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:44,475] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:45,129] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:45,724] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:45,861] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 14:56:45,865] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 14:56:45,919] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 14:56:45,922] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 14:56:46,030] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 14:56:48,005] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 14:56:48,676] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 14:56:49,257] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 14:56:49,897] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 14:56:50,057] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 14:56:50,065] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:50,322] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ttbar_pf +rank0 [2026-10-28 14:56:50,786] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:52,212] INFO : Plotted jets +rank0 [2026-10-28 14:56:53,317] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:53,353] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:53,574] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 14:56:53,948] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:55,900] INFO : Plotted jets +rank0 [2026-10-28 14:56:56,964] INFO : Plotted jet ratio +rank0 [2026-10-28 14:56:56,997] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 14:56:57,201] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_31000/clic_edm_qq_pf +rank0 [2026-10-28 14:56:57,613] INFO : Plotted number of elements +rank0 [2026-10-28 14:56:58,993] INFO : Plotted jets +rank0 [2026-10-28 14:57:00,099] INFO : Plotted jet ratio +rank0 [2026-10-28 14:57:40,686] INFO : Step 31100/50000 rank0 | Train Loss: 8.0023 | Train Loss Unweighted: 2.5771 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.3065 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1116 | Regression_eta: 0.0109 | Regression_pt: 0.1098 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8590 | Classification_binary: 2.3065 | Regression_cos_phi: 0.6045 | Regression_energy: 1.5427 | Regression_eta: 0.5339 | Regression_pt: 1.6521 | Regression_sin_phi: 0.5037 | LR: 3.82e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8277s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2128 | max_loss: 0.0035 | max_backward: 0.3951 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.2830 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5348.0 | uss_MB: 4583.2 | child_rss_MB: 4008.5 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 14:57:40,900] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:58:20,370] INFO : Step 31200/50000 rank0 | Train Loss: 7.6452 | Train Loss Unweighted: 2.6647 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4094 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1061 | Regression_eta: 0.0089 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7630 | Classification_binary: 2.4094 | Regression_cos_phi: 0.4981 | Regression_energy: 1.4666 | Regression_eta: 0.4380 | Regression_pt: 1.5998 | Regression_sin_phi: 0.4702 | LR: 3.79e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2099 | max_optimizer_step: 0.1355 | max_tensorboard_logging: 0.3857 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5348.0 | uss_MB: 4581.1 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 14:58:20,582] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:00,444] INFO : Step 31300/50000 rank0 | Train Loss: 8.6953 | Train Loss Unweighted: 2.7277 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.4206 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1280 | Regression_eta: 0.0114 | Regression_pt: 0.1281 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9230 | Classification_binary: 2.4206 | Regression_cos_phi: 0.6289 | Regression_energy: 1.7694 | Regression_eta: 0.5591 | Regression_pt: 1.9274 | Regression_sin_phi: 0.4669 | LR: 3.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6430s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.2109 | max_optimizer_step: 0.1357 | max_tensorboard_logging: 0.3879 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5348.0 | uss_MB: 4581.2 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 14:59:00,656] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 14:59:40,022] INFO : Step 31400/50000 rank0 | Train Loss: 8.1009 | Train Loss Unweighted: 2.6607 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3561 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1349 | Regression_eta: 0.0078 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7300 | Classification_binary: 2.3561 | Regression_cos_phi: 0.3871 | Regression_energy: 1.8655 | Regression_eta: 0.3847 | Regression_pt: 1.9816 | Regression_sin_phi: 0.3959 | LR: 3.74e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1583 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.2917 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5240.5 | uss_MB: 4473.1 | child_rss_MB: 3987.5 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 14:59:40,237] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:00:20,132] INFO : Step 31500/50000 rank0 | Train Loss: 8.3926 | Train Loss Unweighted: 2.7554 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4649 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1210 | Regression_eta: 0.0092 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7797 | Classification_binary: 2.4649 | Regression_cos_phi: 0.6431 | Regression_energy: 1.6739 | Regression_eta: 0.4525 | Regression_pt: 1.8460 | Regression_sin_phi: 0.5325 | LR: 3.71e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7576s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1074 | max_loss: 0.0037 | max_backward: 0.3929 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2967 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5240.5 | uss_MB: 4475.4 | child_rss_MB: 4015.0 | allocated_MB: 295.1 | reserved_MB: 3746.0 | inactive_split_MB: 86.9 +rank0 [2026-10-28 15:00:20,346] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:00,401] INFO : Step 31600/50000 rank0 | Train Loss: 8.8795 | Train Loss Unweighted: 2.7464 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4256 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1359 | Regression_eta: 0.0111 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8785 | Classification_binary: 2.4256 | Regression_cos_phi: 0.6144 | Regression_energy: 1.8786 | Regression_eta: 0.5434 | Regression_pt: 2.0267 | Regression_sin_phi: 0.5121 | LR: 3.69e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6410s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0854 | max_backward: 0.2034 | max_optimizer_step: 0.1328 | max_tensorboard_logging: 0.3316 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5240.5 | uss_MB: 4473.4 | child_rss_MB: 3986.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:01:00,613] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:01:40,504] INFO : Step 31700/50000 rank0 | Train Loss: 8.0771 | Train Loss Unweighted: 2.5782 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2800 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1284 | Regression_eta: 0.0091 | Regression_pt: 0.1283 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7460 | Classification_binary: 2.2800 | Regression_cos_phi: 0.4495 | Regression_energy: 1.7757 | Regression_eta: 0.4487 | Regression_pt: 1.9300 | Regression_sin_phi: 0.4472 | LR: 3.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6171s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1810 | max_loss: 0.0036 | max_backward: 0.2030 | max_optimizer_step: 0.1359 | max_tensorboard_logging: 0.2867 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:01:40,724] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:02:20,793] INFO : Step 31800/50000 rank0 | Train Loss: 8.4507 | Train Loss Unweighted: 2.7522 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.4542 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1276 | Regression_eta: 0.0104 | Regression_pt: 0.1238 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7872 | Classification_binary: 2.4542 | Regression_cos_phi: 0.5117 | Regression_energy: 1.7650 | Regression_eta: 0.5122 | Regression_pt: 1.8625 | Regression_sin_phi: 0.5579 | LR: 3.64e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6353s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0282 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3083 | max_optimizer_step: 0.1349 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 4005.9 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:02:21,007] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:01,080] INFO : Step 31900/50000 rank0 | Train Loss: 7.9445 | Train Loss Unweighted: 2.6661 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3802 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1239 | Regression_eta: 0.0081 | Regression_pt: 0.1219 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7345 | Classification_binary: 2.3802 | Regression_cos_phi: 0.4251 | Regression_energy: 1.7133 | Regression_eta: 0.3965 | Regression_pt: 1.8340 | Regression_sin_phi: 0.4610 | LR: 3.61e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6382s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.2430 | max_optimizer_step: 0.1329 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5240.5 | uss_MB: 4473.6 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 88.6 +rank0 [2026-10-28 15:03:01,294] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:41,217] INFO : Step 32000/50000 rank0 | Train Loss: 8.5018 | Train Loss Unweighted: 2.6727 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3696 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1288 | Regression_eta: 0.0089 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8339 | Classification_binary: 2.3696 | Regression_cos_phi: 0.7049 | Regression_energy: 1.7806 | Regression_eta: 0.4366 | Regression_pt: 1.9054 | Regression_sin_phi: 0.4707 | LR: 3.58e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6980s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3089 | max_optimizer_step: 0.1340 | max_tensorboard_logging: 0.3804 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5240.7 | uss_MB: 4473.2 | child_rss_MB: 3987.8 | allocated_MB: 291.4 | reserved_MB: 3746.0 | inactive_split_MB: 90.6 +rank0 [2026-10-28 15:03:41,432] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:03:41,545] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-32000.pth +rank0 [2026-10-28 15:03:41,724] INFO : Running validation on rank0 +rank0 [2026-10-28 15:03:41,724] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:03:41,724] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:03:43,016] INFO : saved batch0_epoch32000.parquet +rank0 [2026-10-28 15:03:43,159] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:03:43,289] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:03:43,362] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:03:43,430] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:03:43,498] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:03:43,567] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:03:43,636] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:03:43,750] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:03:43,881] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:03:44,021] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:03:44,091] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:03:44,160] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:03:44,228] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:03:44,298] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:03:44,368] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:03:44,470] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:03:44,476] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:03:44,479] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:03:44,481] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:03:44,483] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:03:44,485] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:03:44,488] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:03:44,489] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:03:44,491] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:03:44,494] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:03:44,495] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:03:44,497] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:03:44,500] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:03:44,501] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:03:44,503] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:03:44,505] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:03:51,801] INFO : VALIDATION | Step=32000/50000 | Train Loss=8.5018 | Valid Loss=8.3395 | Train Loss Unweighted=2.6727 | Valid Loss Unweighted=2.7105 | Stale=1 | ETA=130.1m +rank0 [2026-10-28 15:03:51,811] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:03:51,873] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:03:51,877] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:03:51,995] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:53,982] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:54,653] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:55,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:03:56,023] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:03:56,171] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:03:56,174] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:03:56,231] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:03:56,234] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:03:56,342] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:03:58,280] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:03:58,994] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:03:59,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:04:00,308] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:04:00,459] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:04:00,463] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:04:00,517] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:04:00,520] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:04:00,627] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:04:02,535] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:04:03,116] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:04:03,761] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:04:04,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:04:04,567] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:04:04,586] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:04:04,839] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:04:05,285] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:06,709] INFO : Plotted jets +rank0 [2026-10-28 15:04:07,808] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:07,840] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:04:08,651] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:04:09,024] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:10,401] INFO : Plotted jets +rank0 [2026-10-28 15:04:11,463] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:11,495] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 15:04:11,704] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_32000/clic_edm_qq_pf +rank0 [2026-10-28 15:04:12,115] INFO : Plotted number of elements +rank0 [2026-10-28 15:04:13,502] INFO : Plotted jets +rank0 [2026-10-28 15:04:14,602] INFO : Plotted jet ratio +rank0 [2026-10-28 15:04:55,036] INFO : Step 32100/50000 rank0 | Train Loss: 7.9407 | Train Loss Unweighted: 2.5889 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3074 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1200 | Regression_eta: 0.0083 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7660 | Classification_binary: 2.3074 | Regression_cos_phi: 0.5552 | Regression_energy: 1.6595 | Regression_eta: 0.4071 | Regression_pt: 1.7835 | Regression_sin_phi: 0.4620 | LR: 3.56e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6591s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.1025 | max_backward: 0.2033 | max_optimizer_step: 0.1385 | max_tensorboard_logging: 0.3093 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:04:55,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:05:34,905] INFO : Step 32200/50000 rank0 | Train Loss: 7.9920 | Train Loss Unweighted: 2.6992 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4130 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1228 | Regression_eta: 0.0078 | Regression_pt: 0.1229 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7902 | Classification_binary: 2.4130 | Regression_cos_phi: 0.4839 | Regression_energy: 1.6984 | Regression_eta: 0.3816 | Regression_pt: 1.8488 | Regression_sin_phi: 0.3761 | LR: 3.53e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7124s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0959 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1375 | max_tensorboard_logging: 0.2769 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 4013.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:05:35,119] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:15,601] INFO : Step 32300/50000 rank0 | Train Loss: 8.1707 | Train Loss Unweighted: 2.7288 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4487 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1182 | Regression_eta: 0.0091 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8037 | Classification_binary: 2.4487 | Regression_cos_phi: 0.5587 | Regression_energy: 1.6345 | Regression_eta: 0.4493 | Regression_pt: 1.7423 | Regression_sin_phi: 0.5335 | LR: 3.51e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7584s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1578 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2262 | max_tensorboard_logging: 0.3009 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5438.0 | uss_MB: 4671.3 | child_rss_MB: 3989.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:06:15,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:06:55,027] INFO : Step 32400/50000 rank0 | Train Loss: 7.8144 | Train Loss Unweighted: 2.6143 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3355 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1183 | Regression_eta: 0.0087 | Regression_pt: 0.1201 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7414 | Classification_binary: 2.3355 | Regression_cos_phi: 0.4452 | Regression_energy: 1.6355 | Regression_eta: 0.4262 | Regression_pt: 1.8069 | Regression_sin_phi: 0.4238 | LR: 3.48e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6507s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0324 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3114 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.2799 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5312.8 | uss_MB: 4545.6 | child_rss_MB: 4013.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:06:55,243] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:07:34,657] INFO : Step 32500/50000 rank0 | Train Loss: 8.0604 | Train Loss Unweighted: 2.6533 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.3623 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1248 | Regression_eta: 0.0088 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8262 | Classification_binary: 2.3623 | Regression_cos_phi: 0.4237 | Regression_energy: 1.7255 | Regression_eta: 0.4339 | Regression_pt: 1.8638 | Regression_sin_phi: 0.4252 | LR: 3.46e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6821s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0772 | max_loss: 0.0033 | max_backward: 0.2753 | max_optimizer_step: 0.1353 | max_tensorboard_logging: 0.3810 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.6 | child_rss_MB: 3987.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:07:34,868] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:14,387] INFO : Step 32600/50000 rank0 | Train Loss: 8.3240 | Train Loss Unweighted: 2.6827 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3834 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1272 | Regression_eta: 0.0087 | Regression_pt: 0.1276 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7891 | Classification_binary: 2.3834 | Regression_cos_phi: 0.5349 | Regression_energy: 1.7596 | Regression_eta: 0.4298 | Regression_pt: 1.9190 | Regression_sin_phi: 0.5082 | LR: 3.43e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0288 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3100 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.2841 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.8 | child_rss_MB: 4013.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:08:14,599] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:08:54,658] INFO : Step 32700/50000 rank0 | Train Loss: 8.1799 | Train Loss Unweighted: 2.6389 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3514 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1216 | Regression_eta: 0.0121 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7713 | Classification_binary: 2.3514 | Regression_cos_phi: 0.5153 | Regression_energy: 1.6815 | Regression_eta: 0.5950 | Regression_pt: 1.7974 | Regression_sin_phi: 0.4680 | LR: 3.41e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7210s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0033 | max_backward: 0.3091 | max_optimizer_step: 0.1356 | max_tensorboard_logging: 0.3839 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3988.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:08:54,870] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:09:34,973] INFO : Step 32800/50000 rank0 | Train Loss: 8.7853 | Train Loss Unweighted: 2.7267 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3961 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1436 | Regression_eta: 0.0087 | Regression_pt: 0.1428 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7688 | Classification_binary: 2.3961 | Regression_cos_phi: 0.5319 | Regression_energy: 1.9857 | Regression_eta: 0.4284 | Regression_pt: 2.1485 | Regression_sin_phi: 0.5259 | LR: 3.38e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2152 | max_optimizer_step: 0.1326 | max_tensorboard_logging: 0.3733 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3991.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:09:35,186] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:14,396] INFO : Step 32900/50000 rank0 | Train Loss: 8.1033 | Train Loss Unweighted: 2.6600 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3720 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1240 | Regression_eta: 0.0097 | Regression_pt: 0.1207 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7356 | Classification_binary: 2.3720 | Regression_cos_phi: 0.5337 | Regression_energy: 1.7153 | Regression_eta: 0.4769 | Regression_pt: 1.8164 | Regression_sin_phi: 0.4534 | LR: 3.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6170s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0774 | max_loss: 0.0807 | max_backward: 0.2028 | max_optimizer_step: 0.1325 | max_tensorboard_logging: 0.3171 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5312.8 | uss_MB: 4545.9 | child_rss_MB: 3988.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:10:14,608] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:54,436] INFO : Step 33000/50000 rank0 | Train Loss: 8.3068 | Train Loss Unweighted: 2.6918 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.3953 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1255 | Regression_eta: 0.0090 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7464 | Classification_binary: 2.3953 | Regression_cos_phi: 0.6605 | Regression_energy: 1.7350 | Regression_eta: 0.4426 | Regression_pt: 1.9100 | Regression_sin_phi: 0.4169 | LR: 3.33e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6209s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0904 | max_backward: 0.2031 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3130 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5313.8 | uss_MB: 4545.6 | child_rss_MB: 3984.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:10:54,649] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:10:54,686] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-33000.pth +rank0 [2026-10-28 15:10:54,861] INFO : Running validation on rank0 +rank0 [2026-10-28 15:10:54,862] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:10:54,862] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:10:56,160] INFO : saved batch0_epoch33000.parquet +rank0 [2026-10-28 15:10:56,304] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:10:56,435] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:10:56,508] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:10:56,576] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:10:56,644] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:10:56,713] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:10:56,782] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:10:56,897] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:10:57,028] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:10:57,159] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:10:57,229] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:10:57,298] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:10:57,366] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:10:57,435] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:10:57,505] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:10:57,604] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:10:57,611] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:10:57,614] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:10:57,617] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:10:57,618] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:10:57,619] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:10:57,622] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:10:57,625] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:10:57,627] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:10:57,628] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:10:57,631] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:10:57,632] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:10:57,634] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:10:57,637] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:10:57,639] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:10:57,640] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:11:04,955] INFO : VALIDATION | Step=33000/50000 | Train Loss=8.3068 | Valid Loss=8.1916 | Train Loss Unweighted=2.6918 | Valid Loss Unweighted=2.6784 | Stale=0 | ETA=122.9m +rank0 [2026-10-28 15:11:04,965] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:05,054] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:11:05,074] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:05,265] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:07,336] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:07,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:08,625] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:09,334] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:09,498] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:09,502] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:09,557] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:11:09,560] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:09,671] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:11,591] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:12,193] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:12,884] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:13,476] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:13,645] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:13,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:11:13,702] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:11:13,705] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:11:13,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:11:15,694] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:11:16,274] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:11:16,924] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:11:17,603] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:11:17,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:11:17,785] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:18,041] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:11:18,486] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:19,908] INFO : Plotted jets +rank0 [2026-10-28 15:11:21,549] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:21,583] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:21,806] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:11:22,183] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:23,553] INFO : Plotted jets +rank0 [2026-10-28 15:11:24,623] INFO : Plotted jet ratio +rank0 [2026-10-28 15:11:24,658] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 15:11:24,861] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_33000/clic_edm_qq_pf +rank0 [2026-10-28 15:11:25,273] INFO : Plotted number of elements +rank0 [2026-10-28 15:11:26,659] INFO : Plotted jets +rank0 [2026-10-28 15:11:27,756] INFO : Plotted jet ratio +rank0 [2026-10-28 15:12:07,972] INFO : Step 33100/50000 rank0 | Train Loss: 7.6590 | Train Loss Unweighted: 2.6479 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3785 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1163 | Regression_eta: 0.0070 | Regression_pt: 0.1141 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7784 | Classification_binary: 2.3785 | Regression_cos_phi: 0.4573 | Regression_energy: 1.6077 | Regression_eta: 0.3439 | Regression_pt: 1.7160 | Regression_sin_phi: 0.3772 | LR: 3.31e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6978s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0034 | max_backward: 0.2810 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3850 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3985.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:12:08,184] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:12:48,017] INFO : Step 33200/50000 rank0 | Train Loss: 8.1805 | Train Loss Unweighted: 2.6437 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3472 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1271 | Regression_eta: 0.0095 | Regression_pt: 0.1263 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7584 | Classification_binary: 2.3472 | Regression_cos_phi: 0.4494 | Regression_energy: 1.7571 | Regression_eta: 0.4662 | Regression_pt: 1.9007 | Regression_sin_phi: 0.5016 | LR: 3.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6619s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2331 | max_optimizer_step: 0.1372 | max_tensorboard_logging: 0.3859 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3983.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:12:48,232] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:13:28,132] INFO : Step 33300/50000 rank0 | Train Loss: 8.5542 | Train Loss Unweighted: 2.7013 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3852 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1352 | Regression_eta: 0.0090 | Regression_pt: 0.1366 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8126 | Classification_binary: 2.3852 | Regression_cos_phi: 0.5542 | Regression_energy: 1.8703 | Regression_eta: 0.4407 | Regression_pt: 2.0543 | Regression_sin_phi: 0.4368 | LR: 3.26e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5626s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2037 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.2935 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5511.5 | uss_MB: 4745.1 | child_rss_MB: 3981.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:13:28,347] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:08,415] INFO : Step 33400/50000 rank0 | Train Loss: 7.3038 | Train Loss Unweighted: 2.5144 | Train Loss Components: Classification: 0.0173 | Classification_binary: 2.2665 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1052 | Regression_eta: 0.0077 | Regression_pt: 0.1026 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7261 | Classification_binary: 2.2665 | Regression_cos_phi: 0.4786 | Regression_energy: 1.4547 | Regression_eta: 0.3774 | Regression_pt: 1.5433 | Regression_sin_phi: 0.4572 | LR: 3.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6281s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0957 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2295 | max_tensorboard_logging: 0.2902 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3983.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:14:08,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:14:49,056] INFO : Step 33500/50000 rank0 | Train Loss: 8.9086 | Train Loss Unweighted: 2.7366 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.4129 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1371 | Regression_eta: 0.0105 | Regression_pt: 0.1367 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8640 | Classification_binary: 2.4129 | Regression_cos_phi: 0.6572 | Regression_energy: 1.8964 | Regression_eta: 0.5154 | Regression_pt: 2.0567 | Regression_sin_phi: 0.5058 | LR: 3.21e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1832 | max_loss: 0.0035 | max_backward: 0.2035 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3006 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3986.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:14:49,279] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:15:28,806] INFO : Step 33600/50000 rank0 | Train Loss: 8.1289 | Train Loss Unweighted: 2.7008 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4211 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1166 | Regression_eta: 0.0092 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8177 | Classification_binary: 2.4211 | Regression_cos_phi: 0.5345 | Regression_energy: 1.6119 | Regression_eta: 0.4512 | Regression_pt: 1.7635 | Regression_sin_phi: 0.5291 | LR: 3.19e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6001s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0661 | max_backward: 0.2033 | max_optimizer_step: 0.1343 | max_tensorboard_logging: 0.3111 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5384.6 | uss_MB: 4616.6 | child_rss_MB: 3981.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:15:29,020] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:08,336] INFO : Step 33700/50000 rank0 | Train Loss: 8.2775 | Train Loss Unweighted: 2.6506 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3472 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1300 | Regression_eta: 0.0098 | Regression_pt: 0.1300 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7738 | Classification_binary: 2.3472 | Regression_cos_phi: 0.4315 | Regression_energy: 1.7982 | Regression_eta: 0.4829 | Regression_pt: 1.9553 | Regression_sin_phi: 0.4885 | LR: 3.16e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7089s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1787 | max_loss: 0.0035 | max_backward: 0.2036 | max_optimizer_step: 0.2354 | max_tensorboard_logging: 0.2829 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3986.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:16:08,563] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:16:49,167] INFO : Step 33800/50000 rank0 | Train Loss: 8.1756 | Train Loss Unweighted: 2.6938 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4052 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1237 | Regression_eta: 0.0085 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8041 | Classification_binary: 2.4052 | Regression_cos_phi: 0.5856 | Regression_energy: 1.7112 | Regression_eta: 0.4191 | Regression_pt: 1.8167 | Regression_sin_phi: 0.4336 | LR: 3.14e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6019s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2278 | max_tensorboard_logging: 0.2835 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3984.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:16:49,379] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:17:29,959] INFO : Step 33900/50000 rank0 | Train Loss: 8.7197 | Train Loss Unweighted: 2.7571 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4359 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1380 | Regression_eta: 0.0084 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.4359 | Regression_cos_phi: 0.4954 | Regression_energy: 1.9078 | Regression_eta: 0.4130 | Regression_pt: 2.0701 | Regression_sin_phi: 0.5603 | LR: 3.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7372s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1812 | max_loss: 0.0035 | max_backward: 0.2034 | max_optimizer_step: 0.2362 | max_tensorboard_logging: 0.2920 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 5384.6 | uss_MB: 4616.7 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:17:30,176] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 48.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:18:09,775] INFO : Step 34000/50000 rank0 | Train Loss: 8.3398 | Train Loss Unweighted: 2.6338 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3324 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1288 | Regression_eta: 0.0105 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7968 | Classification_binary: 2.3324 | Regression_cos_phi: 0.5599 | Regression_energy: 1.7812 | Regression_eta: 0.5158 | Regression_pt: 1.9101 | Regression_sin_phi: 0.4437 | LR: 3.09e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0618 | max_backward: 0.2030 | max_optimizer_step: 0.1346 | max_tensorboard_logging: 0.3162 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5385.4 | uss_MB: 4616.6 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:18:09,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:18:10,022] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-34000.pth +rank0 [2026-10-28 15:18:10,197] INFO : Running validation on rank0 +rank0 [2026-10-28 15:18:10,197] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:18:10,197] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:18:11,479] INFO : saved batch0_epoch34000.parquet +rank0 [2026-10-28 15:18:11,623] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:18:11,753] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:18:11,827] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:18:11,896] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:18:11,964] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:18:12,033] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:18:12,101] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:18:12,215] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:18:12,346] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:18:12,477] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:18:12,547] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:18:12,616] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:18:12,684] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:18:12,753] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:18:12,821] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:18:12,922] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:18:12,929] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:18:12,931] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:18:12,932] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:18:12,935] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:18:12,938] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:18:12,939] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:18:12,941] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:18:12,942] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:18:12,945] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:18:12,948] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:18:12,951] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:18:12,952] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:18:12,954] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:18:12,957] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:18:12,958] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:18:20,271] INFO : VALIDATION | Step=34000/50000 | Train Loss=8.3398 | Valid Loss=8.2081 | Train Loss Unweighted=2.6338 | Valid Loss Unweighted=2.6720 | Stale=1 | ETA=115.7m +rank0 [2026-10-28 15:18:20,281] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:20,342] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:18:20,345] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:20,429] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:22,551] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:23,269] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:23,996] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:24,707] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:24,877] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:18:24,880] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:24,936] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:18:24,940] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:25,044] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:26,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:27,530] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:28,131] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:28,781] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:28,935] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:18:28,939] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:18:28,993] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:18:28,996] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:18:29,107] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:18:31,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:18:31,617] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:18:32,266] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:18:32,941] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:18:33,121] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:18:33,129] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:33,386] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:18:33,835] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:35,259] INFO : Plotted jets +rank0 [2026-10-28 15:18:36,903] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:36,937] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:37,162] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:18:37,537] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:38,918] INFO : Plotted jets +rank0 [2026-10-28 15:18:39,981] INFO : Plotted jet ratio +rank0 [2026-10-28 15:18:40,013] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 15:18:40,217] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_34000/clic_edm_qq_pf +rank0 [2026-10-28 15:18:40,632] INFO : Plotted number of elements +rank0 [2026-10-28 15:18:42,020] INFO : Plotted jets +rank0 [2026-10-28 15:18:43,125] INFO : Plotted jet ratio +rank0 [2026-10-28 15:19:23,276] INFO : Step 34100/50000 rank0 | Train Loss: 7.8147 | Train Loss Unweighted: 2.6169 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3510 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1104 | Regression_eta: 0.0091 | Regression_pt: 0.1102 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8478 | Classification_binary: 2.3510 | Regression_cos_phi: 0.4979 | Regression_energy: 1.5270 | Regression_eta: 0.4453 | Regression_pt: 1.6576 | Regression_sin_phi: 0.4881 | LR: 3.06e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.7427s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3035 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.3887 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:19:23,493] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:03,323] INFO : Step 34200/50000 rank0 | Train Loss: 8.2253 | Train Loss Unweighted: 2.5970 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3086 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1214 | Regression_eta: 0.0084 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8513 | Classification_binary: 2.3086 | Regression_cos_phi: 0.6964 | Regression_energy: 1.6793 | Regression_eta: 0.4111 | Regression_pt: 1.7952 | Regression_sin_phi: 0.4835 | LR: 3.04e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6548s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.1020 | max_backward: 0.2031 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:20:03,539] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:20:43,213] INFO : Step 34300/50000 rank0 | Train Loss: 8.6300 | Train Loss Unweighted: 2.7350 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4355 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1252 | Regression_eta: 0.0116 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7899 | Classification_binary: 2.4355 | Regression_cos_phi: 0.6272 | Regression_energy: 1.7314 | Regression_eta: 0.5700 | Regression_pt: 1.8587 | Regression_sin_phi: 0.6172 | LR: 3.02e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0659 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3101 | max_optimizer_step: 0.1398 | max_tensorboard_logging: 0.2948 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5587.7 | uss_MB: 4819.2 | child_rss_MB: 3987.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:20:43,427] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 95.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:21:23,381] INFO : Step 34400/50000 rank0 | Train Loss: 8.7127 | Train Loss Unweighted: 2.6732 | Train Loss Components: Classification: 0.0216 | Classification_binary: 2.3643 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1292 | Regression_eta: 0.0132 | Regression_pt: 0.1273 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.9060 | Classification_binary: 2.3643 | Regression_cos_phi: 0.6291 | Regression_energy: 1.7871 | Regression_eta: 0.6467 | Regression_pt: 1.9154 | Regression_sin_phi: 0.4640 | LR: 2.99e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6575s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0034 | max_backward: 0.2251 | max_optimizer_step: 0.1358 | max_tensorboard_logging: 0.3888 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:21:23,595] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:03,391] INFO : Step 34500/50000 rank0 | Train Loss: 7.5934 | Train Loss Unweighted: 2.5883 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3261 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1112 | Regression_eta: 0.0082 | Regression_pt: 0.1095 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7768 | Classification_binary: 2.3261 | Regression_cos_phi: 0.4518 | Regression_energy: 1.5384 | Regression_eta: 0.4038 | Regression_pt: 1.6472 | Regression_sin_phi: 0.4495 | LR: 2.97e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7627s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.4038 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:22:03,605] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:22:43,089] INFO : Step 34600/50000 rank0 | Train Loss: 8.8270 | Train Loss Unweighted: 2.6202 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2798 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1486 | Regression_eta: 0.0110 | Regression_pt: 0.1470 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7269 | Classification_binary: 2.2798 | Regression_cos_phi: 0.4910 | Regression_energy: 2.0546 | Regression_eta: 0.5385 | Regression_pt: 2.2112 | Regression_sin_phi: 0.5249 | LR: 2.95e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7416s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1841 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2265 | max_tensorboard_logging: 0.2936 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5460.2 | uss_MB: 4692.2 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:22:43,307] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:23:23,554] INFO : Step 34700/50000 rank0 | Train Loss: 8.7026 | Train Loss Unweighted: 2.6519 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3344 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1366 | Regression_eta: 0.0112 | Regression_pt: 0.1323 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.3344 | Regression_cos_phi: 0.6261 | Regression_energy: 1.8885 | Regression_eta: 0.5524 | Regression_pt: 1.9903 | Regression_sin_phi: 0.5147 | LR: 2.92e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7350s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.3824 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 3981.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:23:23,765] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:04,369] INFO : Step 34800/50000 rank0 | Train Loss: 8.5479 | Train Loss Unweighted: 2.6638 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3466 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1376 | Regression_eta: 0.0083 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7799 | Classification_binary: 2.3466 | Regression_cos_phi: 0.5751 | Regression_energy: 1.9027 | Regression_eta: 0.4081 | Regression_pt: 2.0374 | Regression_sin_phi: 0.4982 | LR: 2.90e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1005 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3112 | max_optimizer_step: 0.1351 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.2 | uss_MB: 4692.1 | child_rss_MB: 4005.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:24:04,585] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:24:44,103] INFO : Step 34900/50000 rank0 | Train Loss: 8.1676 | Train Loss Unweighted: 2.6792 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3878 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1260 | Regression_eta: 0.0087 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7842 | Classification_binary: 2.3878 | Regression_cos_phi: 0.5260 | Regression_energy: 1.7422 | Regression_eta: 0.4263 | Regression_pt: 1.8349 | Regression_sin_phi: 0.4663 | LR: 2.88e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1834 | max_loss: 0.0034 | max_backward: 0.2029 | max_optimizer_step: 0.1347 | max_tensorboard_logging: 0.2823 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5460.2 | uss_MB: 4692.2 | child_rss_MB: 3981.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:24:44,321] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:25:23,568] INFO : Step 35000/50000 rank0 | Train Loss: 8.1821 | Train Loss Unweighted: 2.6372 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3414 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1266 | Regression_eta: 0.0088 | Regression_pt: 0.1257 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7987 | Classification_binary: 2.3414 | Regression_cos_phi: 0.5401 | Regression_energy: 1.7502 | Regression_eta: 0.4314 | Regression_pt: 1.8918 | Regression_sin_phi: 0.4283 | LR: 2.85e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6611s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1821 | max_loss: 0.0033 | max_backward: 0.2030 | max_optimizer_step: 0.1338 | max_tensorboard_logging: 0.3360 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5460.1 | uss_MB: 4691.9 | child_rss_MB: 3980.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:25:23,794] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:25:23,812] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-35000.pth +rank0 [2026-10-28 15:25:23,989] INFO : Running validation on rank0 +rank0 [2026-10-28 15:25:23,989] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:25:23,990] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:25:25,322] INFO : saved batch0_epoch35000.parquet +rank0 [2026-10-28 15:25:25,472] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:25:25,604] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:25:25,677] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:25:25,745] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:25:25,814] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:25:25,883] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:25:25,951] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:25:26,063] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:25:26,194] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:25:26,325] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:25:26,396] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:25:26,465] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:25:26,533] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:25:26,602] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:25:26,671] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:25:26,769] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:25:26,776] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:25:26,778] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:25:26,781] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:25:26,782] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:25:26,785] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:25:26,788] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:25:26,789] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:25:26,791] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:25:26,794] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:25:26,795] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:25:26,797] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:25:26,800] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:25:26,803] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:25:26,803] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:25:26,806] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:25:34,075] INFO : VALIDATION | Step=35000/50000 | Train Loss=8.1821 | Valid Loss=8.2301 | Train Loss Unweighted=2.6372 | Valid Loss Unweighted=2.6745 | Stale=2 | ETA=108.4m +rank0 [2026-10-28 15:25:34,087] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:34,149] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:25:34,152] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:34,324] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:36,181] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:36,882] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:37,597] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:38,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:38,508] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:38,511] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:38,567] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:25:38,570] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:38,684] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:40,568] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:41,177] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:41,828] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:42,424] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:42,596] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:25:42,599] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:25:42,654] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:25:42,656] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:25:42,763] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:25:44,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:25:45,265] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:25:45,908] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:25:46,587] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:25:46,753] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:25:46,761] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:47,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:25:47,469] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:48,894] INFO : Plotted jets +rank0 [2026-10-28 15:25:50,002] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:50,042] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:50,264] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:25:50,638] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:52,612] INFO : Plotted jets +rank0 [2026-10-28 15:25:53,730] INFO : Plotted jet ratio +rank0 [2026-10-28 15:25:53,762] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 15:25:53,965] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_35000/clic_edm_qq_pf +rank0 [2026-10-28 15:25:54,382] INFO : Plotted number of elements +rank0 [2026-10-28 15:25:55,771] INFO : Plotted jets +rank0 [2026-10-28 15:25:56,874] INFO : Plotted jet ratio +rank0 [2026-10-28 15:26:37,351] INFO : Step 35100/50000 rank0 | Train Loss: 8.2029 | Train Loss Unweighted: 2.6480 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3447 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1322 | Regression_eta: 0.0087 | Regression_pt: 0.1298 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7579 | Classification_binary: 2.3447 | Regression_cos_phi: 0.4355 | Regression_energy: 1.8284 | Regression_eta: 0.4276 | Regression_pt: 1.9523 | Regression_sin_phi: 0.4566 | LR: 2.83e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6494s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1823 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.1386 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3980.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:26:37,565] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:17,962] INFO : Step 35200/50000 rank0 | Train Loss: 8.5560 | Train Loss Unweighted: 2.6913 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3811 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1344 | Regression_eta: 0.0113 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7679 | Classification_binary: 2.3811 | Regression_cos_phi: 0.6226 | Regression_energy: 1.8591 | Regression_eta: 0.5575 | Regression_pt: 1.9478 | Regression_sin_phi: 0.4201 | LR: 2.81e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6639s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0883 | max_backward: 0.2030 | max_optimizer_step: 0.1374 | max_tensorboard_logging: 0.3340 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3983.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:27:18,178] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:27:57,941] INFO : Step 35300/50000 rank0 | Train Loss: 8.0533 | Train Loss Unweighted: 2.6936 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4137 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1175 | Regression_eta: 0.0085 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8140 | Classification_binary: 2.4137 | Regression_cos_phi: 0.5413 | Regression_energy: 1.6252 | Regression_eta: 0.4184 | Regression_pt: 1.7772 | Regression_sin_phi: 0.4634 | LR: 2.79e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6670s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2260 | max_optimizer_step: 0.1395 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5648.8 | uss_MB: 4882.2 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:27:58,153] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:28:37,814] INFO : Step 35400/50000 rank0 | Train Loss: 7.9015 | Train Loss Unweighted: 2.6882 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.4236 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1107 | Regression_eta: 0.0100 | Regression_pt: 0.1074 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8535 | Classification_binary: 2.4236 | Regression_cos_phi: 0.4753 | Regression_energy: 1.5314 | Regression_eta: 0.4901 | Regression_pt: 1.6157 | Regression_sin_phi: 0.5119 | LR: 2.76e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8383s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0728 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3064 | max_optimizer_step: 0.1424 | max_tensorboard_logging: 0.3061 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5533.6 | uss_MB: 4766.0 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:28:38,028] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:17,548] INFO : Step 35500/50000 rank0 | Train Loss: 7.9987 | Train Loss Unweighted: 2.6968 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4107 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1241 | Regression_eta: 0.0072 | Regression_pt: 0.1213 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8089 | Classification_binary: 2.4107 | Regression_cos_phi: 0.4938 | Regression_energy: 1.7156 | Regression_eta: 0.3540 | Regression_pt: 1.8250 | Regression_sin_phi: 0.3907 | LR: 2.74e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6152s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2154 | max_tensorboard_logging: 0.2889 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5535.6 | uss_MB: 4767.1 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:29:17,761] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:29:57,997] INFO : Step 35600/50000 rank0 | Train Loss: 8.7693 | Train Loss Unweighted: 2.7462 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4352 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1309 | Regression_eta: 0.0124 | Regression_pt: 0.1292 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8781 | Classification_binary: 2.4352 | Regression_cos_phi: 0.6227 | Regression_energy: 1.8103 | Regression_eta: 0.6089 | Regression_pt: 1.9434 | Regression_sin_phi: 0.4707 | LR: 2.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6908s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2552 | max_optimizer_step: 0.1379 | max_tensorboard_logging: 0.3836 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5535.6 | uss_MB: 4767.3 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:29:58,210] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:30:38,036] INFO : Step 35700/50000 rank0 | Train Loss: 8.5567 | Train Loss Unweighted: 2.7770 | Train Loss Components: Classification: 0.0200 | Classification_binary: 2.4760 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1278 | Regression_eta: 0.0107 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8372 | Classification_binary: 2.4760 | Regression_cos_phi: 0.5387 | Regression_energy: 1.7666 | Regression_eta: 0.5234 | Regression_pt: 1.8834 | Regression_sin_phi: 0.5314 | LR: 2.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7255s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2838 | max_optimizer_step: 0.1370 | max_tensorboard_logging: 0.3905 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5535.6 | uss_MB: 4767.3 | child_rss_MB: 3982.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:30:38,251] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:18,051] INFO : Step 35800/50000 rank0 | Train Loss: 7.8628 | Train Loss Unweighted: 2.6751 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.4103 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1111 | Regression_eta: 0.0094 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7634 | Classification_binary: 2.4103 | Regression_cos_phi: 0.5187 | Regression_energy: 1.5359 | Regression_eta: 0.4622 | Regression_pt: 1.6400 | Regression_sin_phi: 0.5323 | LR: 2.68e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8469s | Phase Times: max_pre_step_barrier: 0.1068 | max_batch_to_device: 0.0012 | max_forward: 0.0781 | max_loss: 0.0033 | max_backward: 0.3085 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.3868 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5535.6 | uss_MB: 4767.4 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:31:18,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:31:58,634] INFO : Step 35900/50000 rank0 | Train Loss: 7.8546 | Train Loss Unweighted: 2.6254 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3479 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1187 | Regression_eta: 0.0074 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7651 | Classification_binary: 2.3479 | Regression_cos_phi: 0.4817 | Regression_energy: 1.6408 | Regression_eta: 0.3656 | Regression_pt: 1.7660 | Regression_sin_phi: 0.4874 | LR: 2.65e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6227s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1822 | max_loss: 0.0035 | max_backward: 0.2040 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5535.6 | uss_MB: 4767.4 | child_rss_MB: 3980.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:31:58,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:39,263] INFO : Step 36000/50000 rank0 | Train Loss: 7.7776 | Train Loss Unweighted: 2.5825 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3073 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1176 | Regression_eta: 0.0111 | Regression_pt: 0.1156 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7392 | Classification_binary: 2.3073 | Regression_cos_phi: 0.4414 | Regression_energy: 1.6263 | Regression_eta: 0.5440 | Regression_pt: 1.7390 | Regression_sin_phi: 0.3804 | LR: 2.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6267s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1804 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.2865 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5534.8 | uss_MB: 4766.0 | child_rss_MB: 3980.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:32:39,481] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:32:39,500] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-36000.pth +rank0 [2026-10-28 15:32:39,684] INFO : Running validation on rank0 +rank0 [2026-10-28 15:32:39,684] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:32:39,684] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:32:40,968] INFO : saved batch0_epoch36000.parquet +rank0 [2026-10-28 15:32:41,112] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:32:41,242] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:32:41,315] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:32:41,384] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:32:41,454] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:32:41,523] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:32:41,592] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:32:41,707] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:32:41,838] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:32:41,970] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:32:42,040] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:32:42,109] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:32:42,178] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:32:42,246] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:32:42,316] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:32:42,417] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:32:42,423] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:32:42,426] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:32:42,429] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:32:42,430] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:32:42,432] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:32:42,435] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:32:42,436] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:32:42,438] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:32:42,441] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:32:42,444] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:32:42,445] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:32:42,448] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:32:42,449] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:32:42,451] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:32:42,454] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:32:49,785] INFO : VALIDATION | Step=36000/50000 | Train Loss=7.7776 | Valid Loss=8.1190 | Train Loss Unweighted=2.5825 | Valid Loss Unweighted=2.6484 | Stale=0 | ETA=101.2m +rank0 [2026-10-28 15:32:49,794] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:49,854] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:32:49,857] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:32:50,081] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:52,051] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:52,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:53,350] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:54,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:54,241] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:32:54,245] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:54,301] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:32:54,304] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:32:54,414] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:32:56,203] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:32:56,846] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:32:57,448] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:32:58,090] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:32:58,275] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:32:58,278] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:32:58,333] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:32:58,336] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:32:58,448] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:33:00,381] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:33:00,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:33:01,616] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:33:02,294] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:33:02,479] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:33:02,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:33:02,742] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:33:03,191] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:04,616] INFO : Plotted jets +rank0 [2026-10-28 15:33:05,736] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:05,769] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:33:05,993] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:33:06,370] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:08,311] INFO : Plotted jets +rank0 [2026-10-28 15:33:09,383] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:09,416] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 15:33:09,624] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_36000/clic_edm_qq_pf +rank0 [2026-10-28 15:33:10,036] INFO : Plotted number of elements +rank0 [2026-10-28 15:33:11,420] INFO : Plotted jets +rank0 [2026-10-28 15:33:12,532] INFO : Plotted jet ratio +rank0 [2026-10-28 15:33:53,255] INFO : Step 36100/50000 rank0 | Train Loss: 8.0080 | Train Loss Unweighted: 2.6886 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4015 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1235 | Regression_eta: 0.0092 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7719 | Classification_binary: 2.4015 | Regression_cos_phi: 0.4225 | Regression_energy: 1.7082 | Regression_eta: 0.4527 | Regression_pt: 1.8417 | Regression_sin_phi: 0.4095 | LR: 2.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0765 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3085 | max_optimizer_step: 0.1364 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 4012.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:33:53,466] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:34:33,513] INFO : Step 36200/50000 rank0 | Train Loss: 8.3417 | Train Loss Unweighted: 2.6879 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3976 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1215 | Regression_eta: 0.0129 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8017 | Classification_binary: 2.3976 | Regression_cos_phi: 0.5800 | Regression_energy: 1.6802 | Regression_eta: 0.6319 | Regression_pt: 1.8106 | Regression_sin_phi: 0.4396 | LR: 2.59e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6203s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0767 | max_backward: 0.2034 | max_optimizer_step: 0.1414 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:34:33,726] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:13,717] INFO : Step 36300/50000 rank0 | Train Loss: 7.6702 | Train Loss Unweighted: 2.6941 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.4334 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1107 | Regression_eta: 0.0074 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8087 | Classification_binary: 2.4334 | Regression_cos_phi: 0.4709 | Regression_energy: 1.5314 | Regression_eta: 0.3630 | Regression_pt: 1.6355 | Regression_sin_phi: 0.4273 | LR: 2.57e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7629s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1807 | max_loss: 0.0035 | max_backward: 0.2039 | max_optimizer_step: 0.2451 | max_tensorboard_logging: 0.2907 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5726.3 | uss_MB: 4957.8 | child_rss_MB: 3983.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:13,929] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:35:54,359] INFO : Step 36400/50000 rank0 | Train Loss: 8.1265 | Train Loss Unweighted: 2.7082 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.4306 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1160 | Regression_eta: 0.0093 | Regression_pt: 0.1153 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8135 | Classification_binary: 2.4306 | Regression_cos_phi: 0.5584 | Regression_energy: 1.6046 | Regression_eta: 0.4562 | Regression_pt: 1.7339 | Regression_sin_phi: 0.5293 | LR: 2.55e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.6660s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1058 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2098 | max_tensorboard_logging: 0.3079 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5610.8 | uss_MB: 4841.0 | child_rss_MB: 3982.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:35:54,572] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:36:33,851] INFO : Step 36500/50000 rank0 | Train Loss: 7.5127 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3359 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1141 | Regression_eta: 0.0079 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7155 | Classification_binary: 2.3359 | Regression_cos_phi: 0.4340 | Regression_energy: 1.5775 | Regression_eta: 0.3891 | Regression_pt: 1.7180 | Regression_sin_phi: 0.3426 | LR: 2.52e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0116 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3077 | max_optimizer_step: 0.1393 | max_tensorboard_logging: 0.2905 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5611.8 | uss_MB: 4842.3 | child_rss_MB: 4009.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:36:34,067] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:14,223] INFO : Step 36600/50000 rank0 | Train Loss: 7.5634 | Train Loss Unweighted: 2.5979 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3412 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1082 | Regression_eta: 0.0088 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7725 | Classification_binary: 2.3412 | Regression_cos_phi: 0.4918 | Regression_energy: 1.4958 | Regression_eta: 0.4318 | Regression_pt: 1.6011 | Regression_sin_phi: 0.4293 | LR: 2.50e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8322s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1051 | max_loss: 0.0037 | max_backward: 0.3945 | max_optimizer_step: 0.1427 | max_tensorboard_logging: 0.3485 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5611.8 | uss_MB: 4845.0 | child_rss_MB: 3980.2 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:37:14,437] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:37:54,482] INFO : Step 36700/50000 rank0 | Train Loss: 7.9819 | Train Loss Unweighted: 2.6507 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.3656 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1222 | Regression_eta: 0.0090 | Regression_pt: 0.1217 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7050 | Classification_binary: 2.3656 | Regression_cos_phi: 0.4988 | Regression_energy: 1.6903 | Regression_eta: 0.4405 | Regression_pt: 1.8311 | Regression_sin_phi: 0.4507 | LR: 2.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7500s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1845 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2364 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 5611.8 | uss_MB: 4843.0 | child_rss_MB: 3986.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:37:54,709] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:38:35,347] INFO : Step 36800/50000 rank0 | Train Loss: 8.4863 | Train Loss Unweighted: 2.6280 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.3215 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1297 | Regression_eta: 0.0086 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8425 | Classification_binary: 2.3215 | Regression_cos_phi: 0.4925 | Regression_energy: 1.7940 | Regression_eta: 0.4246 | Regression_pt: 1.9379 | Regression_sin_phi: 0.6734 | LR: 2.46e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.6989s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2756 | max_optimizer_step: 0.1394 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5611.8 | uss_MB: 4843.0 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:38:35,562] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:14,954] INFO : Step 36900/50000 rank0 | Train Loss: 8.0493 | Train Loss Unweighted: 2.5536 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2624 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1246 | Regression_eta: 0.0109 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7332 | Classification_binary: 2.2624 | Regression_cos_phi: 0.5934 | Regression_energy: 1.7236 | Regression_eta: 0.5378 | Regression_pt: 1.8514 | Regression_sin_phi: 0.3476 | LR: 2.44e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7632s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.2119 | max_loss: 0.0035 | max_backward: 0.2885 | max_optimizer_step: 0.1570 | max_tensorboard_logging: 0.2858 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5611.8 | uss_MB: 4845.1 | child_rss_MB: 3983.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:39:15,167] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:54,749] INFO : Step 37000/50000 rank0 | Train Loss: 8.3445 | Train Loss Unweighted: 2.6246 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3143 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1346 | Regression_eta: 0.0099 | Regression_pt: 0.1324 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7942 | Classification_binary: 2.3143 | Regression_cos_phi: 0.4666 | Regression_energy: 1.8608 | Regression_eta: 0.4876 | Regression_pt: 1.9912 | Regression_sin_phi: 0.4299 | LR: 2.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6626s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2137 | max_optimizer_step: 0.1400 | max_tensorboard_logging: 0.3976 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5609.9 | uss_MB: 4841.1 | child_rss_MB: 3979.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:39:54,960] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:39:55,014] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-37000.pth +rank0 [2026-10-28 15:39:55,195] INFO : Running validation on rank0 +rank0 [2026-10-28 15:39:55,196] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:39:55,196] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:39:56,564] INFO : saved batch0_epoch37000.parquet +rank0 [2026-10-28 15:39:56,709] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:39:56,840] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:39:56,914] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:39:56,982] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:39:57,050] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:39:57,119] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:39:57,188] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:39:57,303] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:39:57,434] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:39:57,564] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:39:57,635] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:39:57,705] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:39:57,773] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:39:57,842] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:39:57,912] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:39:58,011] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:39:58,017] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:39:58,020] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:39:58,023] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:39:58,026] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:39:58,027] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:39:58,029] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:39:58,030] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:39:58,033] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:39:58,035] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:39:58,037] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:39:58,039] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:39:58,043] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:39:58,045] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:39:58,046] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:39:58,049] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:40:05,366] INFO : VALIDATION | Step=37000/50000 | Train Loss=8.3445 | Valid Loss=8.0830 | Train Loss Unweighted=2.6246 | Valid Loss Unweighted=2.6450 | Stale=0 | ETA=94.0m +rank0 [2026-10-28 15:40:05,377] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:05,438] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:40:05,441] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:05,613] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:07,515] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:08,217] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:08,933] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:09,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:09,785] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:09,788] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:09,844] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:40:09,847] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:09,956] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:11,733] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:12,434] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:13,035] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:13,683] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:13,848] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:40:13,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:40:13,925] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:40:13,928] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:40:14,022] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:40:15,959] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:40:16,536] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:40:17,179] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:40:17,847] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:40:18,039] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:40:18,046] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:18,304] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:40:18,753] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:20,175] INFO : Plotted jets +rank0 [2026-10-28 15:40:21,883] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:21,915] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:22,144] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:40:22,518] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:23,898] INFO : Plotted jets +rank0 [2026-10-28 15:40:24,958] INFO : Plotted jet ratio +rank0 [2026-10-28 15:40:24,991] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 15:40:25,198] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_37000/clic_edm_qq_pf +rank0 [2026-10-28 15:40:25,611] INFO : Plotted number of elements +rank0 [2026-10-28 15:40:26,995] INFO : Plotted jets +rank0 [2026-10-28 15:40:28,098] INFO : Plotted jet ratio +rank0 [2026-10-28 15:41:07,678] INFO : Step 37100/50000 rank0 | Train Loss: 7.9909 | Train Loss Unweighted: 2.5852 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2988 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1223 | Regression_eta: 0.0099 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7379 | Classification_binary: 2.2988 | Regression_cos_phi: 0.4489 | Regression_energy: 1.6915 | Regression_eta: 0.4885 | Regression_pt: 1.8181 | Regression_sin_phi: 0.5071 | LR: 2.40e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5701s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0113 | max_backward: 0.2028 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3004 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5809.6 | uss_MB: 5041.8 | child_rss_MB: 3978.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:41:07,892] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:41:47,658] INFO : Step 37200/50000 rank0 | Train Loss: 8.2503 | Train Loss Unweighted: 2.7180 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.4268 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1227 | Regression_eta: 0.0087 | Regression_pt: 0.1237 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8284 | Classification_binary: 2.4268 | Regression_cos_phi: 0.5316 | Regression_energy: 1.6962 | Regression_eta: 0.4253 | Regression_pt: 1.8605 | Regression_sin_phi: 0.4814 | LR: 2.38e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6336s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0796 | max_backward: 0.2031 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5809.6 | uss_MB: 5041.9 | child_rss_MB: 3985.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 15:41:47,871] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:42:27,410] INFO : Step 37300/50000 rank0 | Train Loss: 7.3402 | Train Loss Unweighted: 2.5243 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2617 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1134 | Regression_eta: 0.0068 | Regression_pt: 0.1132 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7017 | Classification_binary: 2.2617 | Regression_cos_phi: 0.4013 | Regression_energy: 1.5676 | Regression_eta: 0.3328 | Regression_pt: 1.7031 | Regression_sin_phi: 0.3721 | LR: 2.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6962s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0417 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3048 | max_optimizer_step: 0.1416 | max_tensorboard_logging: 0.2806 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 5809.6 | uss_MB: 5041.9 | child_rss_MB: 4014.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:42:27,622] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:07,076] INFO : Step 37400/50000 rank0 | Train Loss: 7.8503 | Train Loss Unweighted: 2.5392 | Train Loss Components: Classification: 0.0198 | Classification_binary: 2.2656 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1155 | Regression_eta: 0.0092 | Regression_pt: 0.1128 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8279 | Classification_binary: 2.2656 | Regression_cos_phi: 0.5365 | Regression_energy: 1.5966 | Regression_eta: 0.4528 | Regression_pt: 1.6971 | Regression_sin_phi: 0.4738 | LR: 2.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7183s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0783 | max_loss: 0.0037 | max_backward: 0.2398 | max_optimizer_step: 0.1440 | max_tensorboard_logging: 0.4038 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5682.2 | uss_MB: 4913.5 | child_rss_MB: 3984.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:07,290] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:43:47,601] INFO : Step 37500/50000 rank0 | Train Loss: 8.0030 | Train Loss Unweighted: 2.5910 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3061 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1210 | Regression_eta: 0.0085 | Regression_pt: 0.1203 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7763 | Classification_binary: 2.3061 | Regression_cos_phi: 0.5130 | Regression_energy: 1.6739 | Regression_eta: 0.4166 | Regression_pt: 1.8099 | Regression_sin_phi: 0.5072 | LR: 2.32e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8625s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1088 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2890 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5683.2 | uss_MB: 4914.9 | child_rss_MB: 3983.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:43:47,815] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:44:28,383] INFO : Step 37600/50000 rank0 | Train Loss: 8.5621 | Train Loss Unweighted: 2.6155 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3060 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1308 | Regression_eta: 0.0140 | Regression_pt: 0.1286 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8503 | Classification_binary: 2.3060 | Regression_cos_phi: 0.4866 | Regression_energy: 1.8084 | Regression_eta: 0.6858 | Regression_pt: 1.9351 | Regression_sin_phi: 0.4900 | LR: 2.30e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8536s | Phase Times: max_pre_step_barrier: 0.0976 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.3120 | max_optimizer_step: 0.1423 | max_tensorboard_logging: 0.3808 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5684.2 | uss_MB: 4915.1 | child_rss_MB: 3982.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:44:28,596] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:08,795] INFO : Step 37700/50000 rank0 | Train Loss: 8.5215 | Train Loss Unweighted: 2.6826 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3794 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1288 | Regression_eta: 0.0073 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8191 | Classification_binary: 2.3794 | Regression_cos_phi: 0.7821 | Regression_energy: 1.7814 | Regression_eta: 0.3599 | Regression_pt: 1.9141 | Regression_sin_phi: 0.4854 | LR: 2.28e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7930s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.2118 | max_loss: 0.0036 | max_backward: 0.2876 | max_optimizer_step: 0.1528 | max_tensorboard_logging: 0.3036 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 5684.2 | uss_MB: 4917.3 | child_rss_MB: 3983.8 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 15:45:09,008] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:45:49,610] INFO : Step 37800/50000 rank0 | Train Loss: 8.3855 | Train Loss Unweighted: 2.7112 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1257 | Regression_eta: 0.0102 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8711 | Classification_binary: 2.4148 | Regression_cos_phi: 0.5790 | Regression_energy: 1.7387 | Regression_eta: 0.4988 | Regression_pt: 1.8590 | Regression_sin_phi: 0.4241 | LR: 2.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6991s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1641 | max_loss: 0.0036 | max_backward: 0.2032 | max_optimizer_step: 0.2169 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5684.2 | uss_MB: 4915.3 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:45:49,827] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:46:29,652] INFO : Step 37900/50000 rank0 | Train Loss: 8.3340 | Train Loss Unweighted: 2.5603 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.2606 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1273 | Regression_eta: 0.0084 | Regression_pt: 0.1254 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7963 | Classification_binary: 2.2606 | Regression_cos_phi: 0.6635 | Regression_energy: 1.7607 | Regression_eta: 0.4132 | Regression_pt: 1.8870 | Regression_sin_phi: 0.5528 | LR: 2.24e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6332s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0804 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2190 | max_tensorboard_logging: 0.2970 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5684.2 | uss_MB: 4915.4 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:46:29,865] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 42.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:09,230] INFO : Step 38000/50000 rank0 | Train Loss: 8.0994 | Train Loss Unweighted: 2.5566 | Train Loss Components: Classification: 0.0201 | Classification_binary: 2.2575 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1286 | Regression_eta: 0.0073 | Regression_pt: 0.1279 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8421 | Classification_binary: 2.2575 | Regression_cos_phi: 0.4979 | Regression_energy: 1.7777 | Regression_eta: 0.3606 | Regression_pt: 1.9249 | Regression_sin_phi: 0.4387 | LR: 2.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8351s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0824 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3104 | max_optimizer_step: 0.1389 | max_tensorboard_logging: 0.3049 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 5682.1 | uss_MB: 4913.5 | child_rss_MB: 3982.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:47:09,442] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:47:09,461] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-38000.pth +rank0 [2026-10-28 15:47:09,743] INFO : Running validation on rank0 +rank0 [2026-10-28 15:47:09,743] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:47:09,743] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:47:10,926] INFO : saved batch0_epoch38000.parquet +rank0 [2026-10-28 15:47:11,069] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:47:11,200] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:47:11,273] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:47:11,342] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:47:11,411] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:47:11,480] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:47:11,550] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:47:11,663] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:47:11,793] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:47:11,925] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:47:11,996] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:47:12,065] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:47:12,132] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:47:12,201] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:47:12,270] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:47:12,371] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:47:12,473] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:47:12,476] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:47:12,478] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:47:12,481] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:47:12,482] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:47:12,485] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:47:12,488] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:47:12,489] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:47:12,492] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:47:12,493] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:47:12,495] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:47:12,498] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:47:12,501] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:47:12,504] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:47:12,507] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:47:19,881] INFO : VALIDATION | Step=38000/50000 | Train Loss=8.0994 | Valid Loss=8.0654 | Train Loss Unweighted=2.5566 | Valid Loss Unweighted=2.6337 | Stale=0 | ETA=86.8m +rank0 [2026-10-28 15:47:19,892] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:19,954] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:47:19,957] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:20,096] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:22,089] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:22,758] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:23,442] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:24,125] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:24,272] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:24,276] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:24,332] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:47:24,335] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:24,445] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:26,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:26,875] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:27,562] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:28,231] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:28,408] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:47:28,411] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:47:28,465] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:47:28,468] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:47:28,580] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:47:30,317] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:47:30,997] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:47:31,680] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:47:32,355] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:47:32,529] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:47:32,536] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:32,795] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:47:33,245] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:34,671] INFO : Plotted jets +rank0 [2026-10-28 15:47:36,370] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:36,403] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:36,633] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:47:37,006] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:38,388] INFO : Plotted jets +rank0 [2026-10-28 15:47:39,469] INFO : Plotted jet ratio +rank0 [2026-10-28 15:47:39,503] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 15:47:39,711] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_38000/clic_edm_qq_pf +rank0 [2026-10-28 15:47:40,125] INFO : Plotted number of elements +rank0 [2026-10-28 15:47:41,519] INFO : Plotted jets +rank0 [2026-10-28 15:47:42,634] INFO : Plotted jet ratio +rank0 [2026-10-28 15:48:23,373] INFO : Step 38100/50000 rank0 | Train Loss: 7.7936 | Train Loss Unweighted: 2.5981 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.3114 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1244 | Regression_eta: 0.0072 | Regression_pt: 0.1250 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7189 | Classification_binary: 2.3114 | Regression_cos_phi: 0.4470 | Regression_energy: 1.7198 | Regression_eta: 0.3544 | Regression_pt: 1.8806 | Regression_sin_phi: 0.3615 | LR: 2.20e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1069 | max_loss: 0.0036 | max_backward: 0.2038 | max_optimizer_step: 0.2125 | max_tensorboard_logging: 0.2961 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5883.1 | uss_MB: 5114.8 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:48:23,587] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:03,549] INFO : Step 38200/50000 rank0 | Train Loss: 7.8267 | Train Loss Unweighted: 2.5147 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.2350 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1192 | Regression_eta: 0.0112 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8067 | Classification_binary: 2.2350 | Regression_cos_phi: 0.4280 | Regression_energy: 1.6484 | Regression_eta: 0.5515 | Regression_pt: 1.7521 | Regression_sin_phi: 0.4050 | LR: 2.18e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7622s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.1002 | max_forward: 0.0779 | max_loss: 0.0037 | max_backward: 0.3090 | max_optimizer_step: 0.1434 | max_tensorboard_logging: 0.2836 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5883.1 | uss_MB: 5114.9 | child_rss_MB: 4012.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:49:03,763] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:49:43,492] INFO : Step 38300/50000 rank0 | Train Loss: 7.7875 | Train Loss Unweighted: 2.6139 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3447 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1126 | Regression_eta: 0.0089 | Regression_pt: 0.1137 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7817 | Classification_binary: 2.3447 | Regression_cos_phi: 0.5217 | Regression_energy: 1.5564 | Regression_eta: 0.4378 | Regression_pt: 1.7102 | Regression_sin_phi: 0.4350 | LR: 2.16e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7434s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1760 | max_loss: 0.0036 | max_backward: 0.2042 | max_optimizer_step: 0.2117 | max_tensorboard_logging: 0.2952 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 5883.1 | uss_MB: 5114.9 | child_rss_MB: 3984.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:49:43,704] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:50:23,323] INFO : Step 38400/50000 rank0 | Train Loss: 8.2408 | Train Loss Unweighted: 2.6439 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3527 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1233 | Regression_eta: 0.0092 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7712 | Classification_binary: 2.3527 | Regression_cos_phi: 0.6665 | Regression_energy: 1.7053 | Regression_eta: 0.4509 | Regression_pt: 1.8407 | Regression_sin_phi: 0.4536 | LR: 2.14e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7979s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2971 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.4229 | max_tensorboard_flush: 0.0126 | Memory: rss_MB: 5754.2 | uss_MB: 4985.4 | child_rss_MB: 3986.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:50:23,537] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:03,087] INFO : Step 38500/50000 rank0 | Train Loss: 7.8348 | Train Loss Unweighted: 2.6076 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3319 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1173 | Regression_eta: 0.0094 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7822 | Classification_binary: 2.3319 | Regression_cos_phi: 0.5312 | Regression_energy: 1.6222 | Regression_eta: 0.4611 | Regression_pt: 1.7457 | Regression_sin_phi: 0.3604 | LR: 2.12e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6750s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0148 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3104 | max_optimizer_step: 0.1407 | max_tensorboard_logging: 0.2876 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 5755.2 | uss_MB: 4986.2 | child_rss_MB: 4014.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:51:03,303] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:51:43,196] INFO : Step 38600/50000 rank0 | Train Loss: 8.2647 | Train Loss Unweighted: 2.7121 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.4227 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1217 | Regression_eta: 0.0097 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7787 | Classification_binary: 2.4227 | Regression_cos_phi: 0.5987 | Regression_energy: 1.6834 | Regression_eta: 0.4747 | Regression_pt: 1.8383 | Regression_sin_phi: 0.4682 | LR: 2.11e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7288s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1478 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.3091 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5755.2 | uss_MB: 4986.3 | child_rss_MB: 3983.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:51:43,408] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:52:23,599] INFO : Step 38700/50000 rank0 | Train Loss: 8.8647 | Train Loss Unweighted: 2.6434 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2973 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1509 | Regression_eta: 0.0087 | Regression_pt: 0.1521 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7761 | Classification_binary: 2.2973 | Regression_cos_phi: 0.5739 | Regression_energy: 2.0864 | Regression_eta: 0.4292 | Regression_pt: 2.2879 | Regression_sin_phi: 0.4138 | LR: 2.09e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6689s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2138 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3925 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5755.2 | uss_MB: 4986.4 | child_rss_MB: 3980.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:52:23,811] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:03,828] INFO : Step 38800/50000 rank0 | Train Loss: 7.6245 | Train Loss Unweighted: 2.5643 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3039 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1108 | Regression_eta: 0.0084 | Regression_pt: 0.1070 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7351 | Classification_binary: 2.3039 | Regression_cos_phi: 0.5772 | Regression_energy: 1.5319 | Regression_eta: 0.4137 | Regression_pt: 1.6097 | Regression_sin_phi: 0.4529 | LR: 2.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0035 | max_backward: 0.3109 | max_optimizer_step: 0.1376 | max_tensorboard_logging: 0.3917 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5755.2 | uss_MB: 4986.6 | child_rss_MB: 3982.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 15:53:04,040] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:53:43,808] INFO : Step 38900/50000 rank0 | Train Loss: 8.1884 | Train Loss Unweighted: 2.6882 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3896 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1280 | Regression_eta: 0.0084 | Regression_pt: 0.1293 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7737 | Classification_binary: 2.3896 | Regression_cos_phi: 0.4837 | Regression_energy: 1.7696 | Regression_eta: 0.4134 | Regression_pt: 1.9453 | Regression_sin_phi: 0.4132 | LR: 2.05e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7467s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3057 | max_optimizer_step: 0.1402 | max_tensorboard_logging: 0.3753 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 5755.2 | uss_MB: 4986.6 | child_rss_MB: 3985.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:53:44,021] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:23,982] INFO : Step 39000/50000 rank0 | Train Loss: 7.7432 | Train Loss Unweighted: 2.5425 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.2664 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1170 | Regression_eta: 0.0090 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8016 | Classification_binary: 2.2664 | Regression_cos_phi: 0.4368 | Regression_energy: 1.6173 | Regression_eta: 0.4445 | Regression_pt: 1.7629 | Regression_sin_phi: 0.4136 | LR: 2.03e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.5741s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0039 | max_backward: 0.2033 | max_optimizer_step: 0.1373 | max_tensorboard_logging: 0.3279 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5753.8 | uss_MB: 4985.2 | child_rss_MB: 3985.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:54:24,196] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:54:24,218] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-39000.pth +rank0 [2026-10-28 15:54:24,396] INFO : Running validation on rank0 +rank0 [2026-10-28 15:54:24,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 15:54:24,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 15:54:25,675] INFO : saved batch0_epoch39000.parquet +rank0 [2026-10-28 15:54:25,821] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 15:54:25,951] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 15:54:26,025] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 15:54:26,093] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 15:54:26,160] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 15:54:26,230] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 15:54:26,299] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 15:54:26,413] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 15:54:26,544] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 15:54:26,675] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 15:54:26,745] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 15:54:26,814] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 15:54:26,884] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 15:54:26,952] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 15:54:27,024] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 15:54:27,125] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 15:54:27,132] INFO : plotted pt_target histogram +rank0 [2026-10-28 15:54:27,135] INFO : plotted pt_pred histogram +rank0 [2026-10-28 15:54:27,136] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 15:54:27,139] INFO : plotted eta_target histogram +rank0 [2026-10-28 15:54:27,142] INFO : plotted eta_pred histogram +rank0 [2026-10-28 15:54:27,144] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 15:54:27,147] INFO : plotted sphi_target histogram +rank0 [2026-10-28 15:54:27,150] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 15:54:27,153] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 15:54:27,154] INFO : plotted cphi_target histogram +rank0 [2026-10-28 15:54:27,157] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 15:54:27,159] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 15:54:27,160] INFO : plotted energy_target histogram +rank0 [2026-10-28 15:54:27,163] INFO : plotted energy_pred histogram +rank0 [2026-10-28 15:54:27,166] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 15:54:34,493] INFO : VALIDATION | Step=39000/50000 | Train Loss=7.7432 | Valid Loss=8.1049 | Train Loss Unweighted=2.5425 | Valid Loss Unweighted=2.6448 | Stale=1 | ETA=79.5m +rank0 [2026-10-28 15:54:34,503] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:34,596] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 15:54:34,600] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:34,805] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:36,722] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:37,422] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:38,133] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:38,844] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:39,008] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 15:54:39,011] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:39,066] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 15:54:39,069] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:39,184] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:41,029] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:41,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:42,259] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:42,904] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:43,100] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:54:43,103] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 15:54:43,157] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 15:54:43,160] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 15:54:43,278] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 15:54:44,993] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 15:54:45,651] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 15:54:46,344] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 15:54:47,016] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 15:54:47,191] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 15:54:47,199] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:47,454] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ttbar_pf +rank0 [2026-10-28 15:54:47,901] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:49,335] INFO : Plotted jets +rank0 [2026-10-28 15:54:50,454] INFO : Plotted jet ratio +rank0 [2026-10-28 15:54:50,486] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:50,712] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 15:54:51,088] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:53,037] INFO : Plotted jets +rank0 [2026-10-28 15:54:54,106] INFO : Plotted jet ratio +rank0 [2026-10-28 15:54:54,139] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 15:54:54,340] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_39000/clic_edm_qq_pf +rank0 [2026-10-28 15:54:54,759] INFO : Plotted number of elements +rank0 [2026-10-28 15:54:56,150] INFO : Plotted jets +rank0 [2026-10-28 15:54:57,259] INFO : Plotted jet ratio +rank0 [2026-10-28 15:55:37,171] INFO : Step 39100/50000 rank0 | Train Loss: 7.7673 | Train Loss Unweighted: 2.5947 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3245 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1154 | Regression_eta: 0.0090 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7848 | Classification_binary: 2.3245 | Regression_cos_phi: 0.5179 | Regression_energy: 1.5958 | Regression_eta: 0.4441 | Regression_pt: 1.6847 | Regression_sin_phi: 0.4156 | LR: 2.01e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6576s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0235 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3073 | max_optimizer_step: 0.1409 | max_tensorboard_logging: 0.2743 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 4015.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:55:37,386] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:16,671] INFO : Step 39200/50000 rank0 | Train Loss: 7.9284 | Train Loss Unweighted: 2.5878 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3093 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1173 | Regression_eta: 0.0095 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7648 | Classification_binary: 2.3093 | Regression_cos_phi: 0.5089 | Regression_energy: 1.6225 | Regression_eta: 0.4690 | Regression_pt: 1.7625 | Regression_sin_phi: 0.4915 | LR: 2.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7801s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0248 | max_forward: 0.0771 | max_loss: 0.0033 | max_backward: 0.3112 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2832 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 3983.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:56:16,883] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:56:56,424] INFO : Step 39300/50000 rank0 | Train Loss: 8.3931 | Train Loss Unweighted: 2.6486 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3367 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1340 | Regression_eta: 0.0090 | Regression_pt: 0.1353 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7401 | Classification_binary: 2.3367 | Regression_cos_phi: 0.5352 | Regression_energy: 1.8532 | Regression_eta: 0.4399 | Regression_pt: 2.0361 | Regression_sin_phi: 0.4520 | LR: 1.98e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7440s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1790 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2351 | max_tensorboard_logging: 0.2803 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 3985.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:56:56,638] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:57:35,963] INFO : Step 39400/50000 rank0 | Train Loss: 8.3117 | Train Loss Unweighted: 2.6444 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3441 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1273 | Regression_eta: 0.0092 | Regression_pt: 0.1287 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7322 | Classification_binary: 2.3441 | Regression_cos_phi: 0.5650 | Regression_energy: 1.7607 | Regression_eta: 0.4515 | Regression_pt: 1.9358 | Regression_sin_phi: 0.5224 | LR: 1.96e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6803s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0157 | max_forward: 0.0774 | max_loss: 0.0033 | max_backward: 0.3073 | max_optimizer_step: 0.1426 | max_tensorboard_logging: 0.2808 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5934.9 | uss_MB: 5168.8 | child_rss_MB: 4011.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 15:57:36,175] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:15,383] INFO : Step 39500/50000 rank0 | Train Loss: 7.8927 | Train Loss Unweighted: 2.6213 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3375 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1217 | Regression_eta: 0.0073 | Regression_pt: 0.1214 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7980 | Classification_binary: 2.3375 | Regression_cos_phi: 0.4761 | Regression_energy: 1.6826 | Regression_eta: 0.3567 | Regression_pt: 1.8265 | Regression_sin_phi: 0.4152 | LR: 1.94e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7497s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1454 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0087 | Memory: rss_MB: 5826.9 | uss_MB: 5060.2 | child_rss_MB: 3985.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:58:15,597] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:58:55,527] INFO : Step 39600/50000 rank0 | Train Loss: 7.5867 | Train Loss Unweighted: 2.5958 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3279 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1136 | Regression_eta: 0.0082 | Regression_pt: 0.1146 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8227 | Classification_binary: 2.3279 | Regression_cos_phi: 0.3909 | Regression_energy: 1.5704 | Regression_eta: 0.4026 | Regression_pt: 1.7239 | Regression_sin_phi: 0.3481 | LR: 1.93e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7334s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0775 | max_loss: 0.0035 | max_backward: 0.3016 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.3704 | max_tensorboard_flush: 0.0095 | Memory: rss_MB: 5826.9 | uss_MB: 5060.4 | child_rss_MB: 3986.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 15:58:55,736] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 15:59:35,507] INFO : Step 39700/50000 rank0 | Train Loss: 7.6111 | Train Loss Unweighted: 2.6369 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.3809 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1063 | Regression_eta: 0.0095 | Regression_pt: 0.1068 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7968 | Classification_binary: 2.3809 | Regression_cos_phi: 0.4977 | Regression_energy: 1.4698 | Regression_eta: 0.4687 | Regression_pt: 1.6071 | Regression_sin_phi: 0.3902 | LR: 1.91e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7439s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3097 | max_optimizer_step: 0.1420 | max_tensorboard_logging: 0.3814 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5826.9 | uss_MB: 5060.5 | child_rss_MB: 3986.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 15:59:35,722] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:16,036] INFO : Step 39800/50000 rank0 | Train Loss: 8.4719 | Train Loss Unweighted: 2.7243 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.4189 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1300 | Regression_eta: 0.0076 | Regression_pt: 0.1310 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7925 | Classification_binary: 2.4189 | Regression_cos_phi: 0.6634 | Regression_energy: 1.7972 | Regression_eta: 0.3712 | Regression_pt: 1.9708 | Regression_sin_phi: 0.4577 | LR: 1.89e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6630s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1357 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.2102 | max_tensorboard_logging: 0.2840 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 5826.9 | uss_MB: 5060.6 | child_rss_MB: 3983.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:16,250] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:00:56,451] INFO : Step 39900/50000 rank0 | Train Loss: 8.0349 | Train Loss Unweighted: 2.6589 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3860 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1144 | Regression_eta: 0.0107 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7612 | Classification_binary: 2.3860 | Regression_cos_phi: 0.5875 | Regression_energy: 1.5817 | Regression_eta: 0.5243 | Regression_pt: 1.6853 | Regression_sin_phi: 0.5089 | LR: 1.88e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7573s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3039 | max_optimizer_step: 0.1403 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 5826.9 | uss_MB: 5060.7 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:00:56,663] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:36,356] INFO : Step 40000/50000 rank0 | Train Loss: 8.4132 | Train Loss Unweighted: 2.6903 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.3896 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1267 | Regression_eta: 0.0101 | Regression_pt: 0.1272 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.3896 | Regression_cos_phi: 0.5595 | Regression_energy: 1.7515 | Regression_eta: 0.4964 | Regression_pt: 1.9134 | Regression_sin_phi: 0.4352 | LR: 1.86e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7614s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3097 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2872 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5826.9 | uss_MB: 5060.7 | child_rss_MB: 3981.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:01:36,567] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:01:36,585] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-40000.pth +rank0 [2026-10-28 16:01:36,865] INFO : Running validation on rank0 +rank0 [2026-10-28 16:01:36,865] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:01:36,865] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:01:38,059] INFO : saved batch0_epoch40000.parquet +rank0 [2026-10-28 16:01:38,202] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:01:38,334] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:01:38,407] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:01:38,476] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:01:38,544] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:01:38,613] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:01:38,682] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:01:38,796] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:01:38,927] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:01:39,059] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:01:39,131] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:01:39,200] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:01:39,268] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:01:39,337] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:01:39,406] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:01:39,506] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:01:39,609] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:01:39,612] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:01:39,615] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:01:39,617] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:01:39,619] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:01:39,622] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:01:39,624] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:01:39,625] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:01:39,628] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:01:39,629] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:01:39,632] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:01:39,635] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:01:39,637] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:01:39,638] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:01:39,641] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:01:46,951] INFO : VALIDATION | Step=40000/50000 | Train Loss=8.4132 | Valid Loss=8.0601 | Train Loss Unweighted=2.6903 | Valid Loss Unweighted=2.6280 | Stale=0 | ETA=72.3m +rank0 [2026-10-28 16:01:46,960] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:47,020] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:01:47,024] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:01:47,198] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:49,205] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:49,866] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:50,570] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:51,241] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:51,454] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:51,458] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:51,513] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:01:51,516] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:01:51,628] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:53,469] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:54,174] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:54,786] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:55,432] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:55,621] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:55,624] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:01:55,678] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:01:55,681] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:01:55,797] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:01:57,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:01:58,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:01:58,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:01:59,605] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:01:59,760] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:01:59,768] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:02:00,019] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:02:00,469] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:01,899] INFO : Plotted jets +rank0 [2026-10-28 16:02:03,590] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:03,621] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:03,850] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:02:04,224] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:05,608] INFO : Plotted jets +rank0 [2026-10-28 16:02:06,685] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:06,717] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 16:02:06,922] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_40000/clic_edm_qq_pf +rank0 [2026-10-28 16:02:07,335] INFO : Plotted number of elements +rank0 [2026-10-28 16:02:08,726] INFO : Plotted jets +rank0 [2026-10-28 16:02:09,836] INFO : Plotted jet ratio +rank0 [2026-10-28 16:02:49,847] INFO : Step 40100/50000 rank0 | Train Loss: 7.9511 | Train Loss Unweighted: 2.5532 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.2735 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1186 | Regression_eta: 0.0092 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7573 | Classification_binary: 2.2735 | Regression_cos_phi: 0.6014 | Regression_energy: 1.6405 | Regression_eta: 0.4498 | Regression_pt: 1.7511 | Regression_sin_phi: 0.4775 | LR: 1.84e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7737s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1843 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2497 | max_tensorboard_logging: 0.2843 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6028.5 | uss_MB: 5264.4 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:02:50,062] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:03:30,075] INFO : Step 40200/50000 rank0 | Train Loss: 7.8004 | Train Loss Unweighted: 2.5651 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2961 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1124 | Regression_eta: 0.0093 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7838 | Classification_binary: 2.2961 | Regression_cos_phi: 0.5573 | Regression_energy: 1.5542 | Regression_eta: 0.4546 | Regression_pt: 1.6863 | Regression_sin_phi: 0.4681 | LR: 1.83e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.8227s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0550 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1426 | max_tensorboard_logging: 0.2792 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6028.5 | uss_MB: 5264.5 | child_rss_MB: 3985.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:03:30,287] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:10,914] INFO : Step 40300/50000 rank0 | Train Loss: 7.8063 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3289 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1159 | Regression_eta: 0.0087 | Regression_pt: 0.1142 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7828 | Classification_binary: 2.3289 | Regression_cos_phi: 0.5544 | Regression_energy: 1.6025 | Regression_eta: 0.4280 | Regression_pt: 1.7181 | Regression_sin_phi: 0.3916 | LR: 1.81e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.5404s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0067 | max_backward: 0.2038 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.2828 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6028.5 | uss_MB: 5264.5 | child_rss_MB: 3985.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:04:11,129] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 145.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:04:50,339] INFO : Step 40400/50000 rank0 | Train Loss: 7.9823 | Train Loss Unweighted: 2.5962 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3061 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1263 | Regression_eta: 0.0088 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7754 | Classification_binary: 2.3061 | Regression_cos_phi: 0.4104 | Regression_energy: 1.7466 | Regression_eta: 0.4345 | Regression_pt: 1.8353 | Regression_sin_phi: 0.4740 | LR: 1.79e-04 | DataLoad Time: 0.0024s | Model Forward Time: 1.7697s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.1046 | max_loss: 0.0744 | max_backward: 0.2885 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.2764 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6028.5 | uss_MB: 5266.6 | child_rss_MB: 3982.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:04:50,558] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:05:30,766] INFO : Step 40500/50000 rank0 | Train Loss: 8.3412 | Train Loss Unweighted: 2.6182 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3144 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1295 | Regression_eta: 0.0105 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8055 | Classification_binary: 2.3144 | Regression_cos_phi: 0.5977 | Regression_energy: 1.7912 | Regression_eta: 0.5141 | Regression_pt: 1.9396 | Regression_sin_phi: 0.3786 | LR: 1.78e-04 | DataLoad Time: 0.0031s | Model Forward Time: 1.7701s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0020 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3093 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3918 | max_tensorboard_flush: 0.0013 | Memory: rss_MB: 5897.9 | uss_MB: 5133.5 | child_rss_MB: 3983.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:05:30,978] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:11,347] INFO : Step 40600/50000 rank0 | Train Loss: 7.8341 | Train Loss Unweighted: 2.6014 | Train Loss Components: Classification: 0.0202 | Classification_binary: 2.3248 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1174 | Regression_eta: 0.0081 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8461 | Classification_binary: 2.3248 | Regression_cos_phi: 0.4745 | Regression_energy: 1.6233 | Regression_eta: 0.3997 | Regression_pt: 1.7530 | Regression_sin_phi: 0.4127 | LR: 1.76e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7665s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1041 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3099 | max_optimizer_step: 0.1431 | max_tensorboard_logging: 0.2837 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5897.9 | uss_MB: 5133.5 | child_rss_MB: 4008.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:06:11,560] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:06:51,240] INFO : Step 40700/50000 rank0 | Train Loss: 7.4236 | Train Loss Unweighted: 2.5730 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3109 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1129 | Regression_eta: 0.0059 | Regression_pt: 0.1123 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7422 | Classification_binary: 2.3109 | Regression_cos_phi: 0.4629 | Regression_energy: 1.5613 | Regression_eta: 0.2876 | Regression_pt: 1.6888 | Regression_sin_phi: 0.3700 | LR: 1.75e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6301s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0037 | max_backward: 0.2038 | max_optimizer_step: 0.2105 | max_tensorboard_logging: 0.2804 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3979.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:06:51,455] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:07:31,033] INFO : Step 40800/50000 rank0 | Train Loss: 8.0254 | Train Loss Unweighted: 2.6312 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.3607 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1133 | Regression_eta: 0.0100 | Regression_pt: 0.1087 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8528 | Classification_binary: 2.3607 | Regression_cos_phi: 0.5663 | Regression_energy: 1.5666 | Regression_eta: 0.4900 | Regression_pt: 1.6353 | Regression_sin_phi: 0.5538 | LR: 1.73e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0964 | max_backward: 0.2032 | max_optimizer_step: 0.1439 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:07:31,246] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:11,386] INFO : Step 40900/50000 rank0 | Train Loss: 7.8040 | Train Loss Unweighted: 2.6016 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.3293 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1182 | Regression_eta: 0.0089 | Regression_pt: 0.1116 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7306 | Classification_binary: 2.3293 | Regression_cos_phi: 0.5172 | Regression_energy: 1.6348 | Regression_eta: 0.4397 | Regression_pt: 1.6796 | Regression_sin_phi: 0.4728 | LR: 1.72e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6957s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0037 | max_backward: 0.2709 | max_optimizer_step: 0.1392 | max_tensorboard_logging: 0.3727 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 5897.9 | uss_MB: 5133.9 | child_rss_MB: 3981.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:08:11,601] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:52,069] INFO : Step 41000/50000 rank0 | Train Loss: 8.2526 | Train Loss Unweighted: 2.7058 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.4090 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1263 | Regression_eta: 0.0091 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7721 | Classification_binary: 2.4090 | Regression_cos_phi: 0.5069 | Regression_energy: 1.7460 | Regression_eta: 0.4495 | Regression_pt: 1.9187 | Regression_sin_phi: 0.4504 | LR: 1.70e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7651s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3105 | max_optimizer_step: 0.1448 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 5897.9 | uss_MB: 5134.0 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:08:52,286] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:08:52,386] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-41000.pth +rank0 [2026-10-28 16:08:52,562] INFO : Running validation on rank0 +rank0 [2026-10-28 16:08:52,562] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:08:52,562] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:08:53,843] INFO : saved batch0_epoch41000.parquet +rank0 [2026-10-28 16:08:53,986] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:08:54,118] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:08:54,191] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:08:54,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:08:54,329] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:08:54,398] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:08:54,466] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:08:54,582] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:08:54,714] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:08:54,845] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:08:54,916] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:08:54,985] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:08:55,053] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:08:55,122] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:08:55,191] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:08:55,291] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:08:55,297] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:08:55,300] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:08:55,303] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:08:55,303] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:08:55,306] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:08:55,308] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:08:55,310] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:08:55,311] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:08:55,314] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:08:55,317] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:08:55,319] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:08:55,320] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:08:55,323] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:08:55,323] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:08:55,327] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:09:02,639] INFO : VALIDATION | Step=41000/50000 | Train Loss=8.2526 | Valid Loss=8.0337 | Train Loss Unweighted=2.7058 | Valid Loss Unweighted=2.6288 | Stale=0 | ETA=65.1m +rank0 [2026-10-28 16:09:02,649] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:02,711] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:09:02,715] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:02,808] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:04,804] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:05,481] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:06,170] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:06,845] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:07,007] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:07,010] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:07,068] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:09:07,071] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:07,185] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:08,938] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:09,602] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:10,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:10,967] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:11,165] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:09:11,168] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:09:11,223] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:09:11,226] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:09:11,343] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:09:13,046] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:09:13,708] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:09:14,399] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:09:15,082] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:09:15,281] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:09:15,289] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:15,543] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:09:15,992] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:17,418] INFO : Plotted jets +rank0 [2026-10-28 16:09:18,545] INFO : Plotted jet ratio +rank0 [2026-10-28 16:09:18,577] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:18,802] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:09:19,174] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:21,187] INFO : Plotted jets +rank0 [2026-10-28 16:09:22,256] INFO : Plotted jet ratio +rank0 [2026-10-28 16:09:22,288] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 16:09:22,493] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_41000/clic_edm_qq_pf +rank0 [2026-10-28 16:09:22,908] INFO : Plotted number of elements +rank0 [2026-10-28 16:09:24,291] INFO : Plotted jets +rank0 [2026-10-28 16:09:25,397] INFO : Plotted jet ratio +rank0 [2026-10-28 16:10:06,437] INFO : Step 41100/50000 rank0 | Train Loss: 8.6334 | Train Loss Unweighted: 2.6600 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.3525 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1300 | Regression_eta: 0.0101 | Regression_pt: 0.1278 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8333 | Classification_binary: 2.3525 | Regression_cos_phi: 0.7147 | Regression_energy: 1.7971 | Regression_eta: 0.4983 | Regression_pt: 1.9221 | Regression_sin_phi: 0.5154 | LR: 1.69e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7252s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0774 | max_loss: 0.0036 | max_backward: 0.2466 | max_optimizer_step: 0.1437 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6089.4 | uss_MB: 5322.7 | child_rss_MB: 3976.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:10:06,650] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:10:46,396] INFO : Step 41200/50000 rank0 | Train Loss: 8.3401 | Train Loss Unweighted: 2.6864 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.3950 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1246 | Regression_eta: 0.0112 | Regression_pt: 0.1192 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7746 | Classification_binary: 2.3950 | Regression_cos_phi: 0.5924 | Regression_energy: 1.7235 | Regression_eta: 0.5495 | Regression_pt: 1.7939 | Regression_sin_phi: 0.5113 | LR: 1.67e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.5698s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1418 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6089.4 | uss_MB: 5322.7 | child_rss_MB: 3975.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:10:46,610] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:11:26,285] INFO : Step 41300/50000 rank0 | Train Loss: 7.1674 | Train Loss Unweighted: 2.4152 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.1597 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1092 | Regression_eta: 0.0061 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7066 | Classification_binary: 2.1597 | Regression_cos_phi: 0.4536 | Regression_energy: 1.5102 | Regression_eta: 0.3020 | Regression_pt: 1.6500 | Regression_sin_phi: 0.3854 | LR: 1.66e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9615s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0704 | max_forward: 0.1051 | max_loss: 0.0037 | max_backward: 0.3941 | max_optimizer_step: 0.1451 | max_tensorboard_logging: 0.3827 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6089.4 | uss_MB: 5324.8 | child_rss_MB: 3977.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:11:26,498] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:06,266] INFO : Step 41400/50000 rank0 | Train Loss: 8.2236 | Train Loss Unweighted: 2.6999 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.4138 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1199 | Regression_eta: 0.0106 | Regression_pt: 0.1200 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7837 | Classification_binary: 2.4138 | Regression_cos_phi: 0.5174 | Regression_energy: 1.6579 | Regression_eta: 0.5231 | Regression_pt: 1.8049 | Regression_sin_phi: 0.5228 | LR: 1.64e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6610s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0776 | max_loss: 0.0035 | max_backward: 0.2075 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3697 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6089.4 | uss_MB: 5322.8 | child_rss_MB: 3979.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:12:06,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:12:46,717] INFO : Step 41500/50000 rank0 | Train Loss: 8.1958 | Train Loss Unweighted: 2.5559 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.2544 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1299 | Regression_eta: 0.0093 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8675 | Classification_binary: 2.2544 | Regression_cos_phi: 0.4581 | Regression_energy: 1.7958 | Regression_eta: 0.4574 | Regression_pt: 1.9080 | Regression_sin_phi: 0.4545 | LR: 1.63e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.9596s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1900 | max_loss: 0.0035 | max_backward: 0.3938 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.2875 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5212.0 | child_rss_MB: 3979.4 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 16:12:46,930] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:13:27,100] INFO : Step 41600/50000 rank0 | Train Loss: 8.0937 | Train Loss Unweighted: 2.5803 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1264 | Regression_eta: 0.0085 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7820 | Classification_binary: 2.2874 | Regression_cos_phi: 0.5610 | Regression_energy: 1.7474 | Regression_eta: 0.4189 | Regression_pt: 1.8532 | Regression_sin_phi: 0.4437 | LR: 1.61e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7819s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1804 | max_loss: 0.0036 | max_backward: 0.2035 | max_optimizer_step: 0.2458 | max_tensorboard_logging: 0.2908 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 5978.1 | uss_MB: 5210.4 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:13:27,313] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:07,679] INFO : Step 41700/50000 rank0 | Train Loss: 7.8485 | Train Loss Unweighted: 2.6543 | Train Loss Components: Classification: 0.0157 | Classification_binary: 2.3789 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1180 | Regression_eta: 0.0074 | Regression_pt: 0.1174 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6587 | Classification_binary: 2.3789 | Regression_cos_phi: 0.5719 | Regression_energy: 1.6317 | Regression_eta: 0.3617 | Regression_pt: 1.7659 | Regression_sin_phi: 0.4797 | LR: 1.60e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.7465s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1835 | max_loss: 0.0035 | max_backward: 0.2042 | max_optimizer_step: 0.2315 | max_tensorboard_logging: 0.2826 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 5978.1 | uss_MB: 5210.5 | child_rss_MB: 3977.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:14:07,893] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:14:47,863] INFO : Step 41800/50000 rank0 | Train Loss: 8.0963 | Train Loss Unweighted: 2.6075 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3219 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1204 | Regression_eta: 0.0092 | Regression_pt: 0.1198 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8046 | Classification_binary: 2.3219 | Regression_cos_phi: 0.5933 | Regression_energy: 1.6653 | Regression_eta: 0.4503 | Regression_pt: 1.8030 | Regression_sin_phi: 0.4578 | LR: 1.58e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7859s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0206 | max_forward: 0.0776 | max_loss: 0.0036 | max_backward: 0.3103 | max_optimizer_step: 0.1417 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5210.6 | child_rss_MB: 3978.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:14:48,077] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:15:28,624] INFO : Step 41900/50000 rank0 | Train Loss: 8.3514 | Train Loss Unweighted: 2.6263 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3235 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1295 | Regression_eta: 0.0103 | Regression_pt: 0.1277 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7850 | Classification_binary: 2.3235 | Regression_cos_phi: 0.5657 | Regression_energy: 1.7912 | Regression_eta: 0.5052 | Regression_pt: 1.9211 | Regression_sin_phi: 0.4597 | LR: 1.57e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1788 | max_loss: 0.0037 | max_backward: 0.2040 | max_optimizer_step: 0.2130 | max_tensorboard_logging: 0.2821 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 5978.1 | uss_MB: 5210.7 | child_rss_MB: 3979.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:15:28,840] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:08,707] INFO : Step 42000/50000 rank0 | Train Loss: 8.2783 | Train Loss Unweighted: 2.7126 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.4018 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1365 | Regression_eta: 0.0070 | Regression_pt: 0.1354 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7666 | Classification_binary: 2.4018 | Regression_cos_phi: 0.4328 | Regression_energy: 1.8872 | Regression_eta: 0.3416 | Regression_pt: 2.0377 | Regression_sin_phi: 0.4105 | LR: 1.56e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7029s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1060 | max_loss: 0.0035 | max_backward: 0.2037 | max_optimizer_step: 0.2155 | max_tensorboard_logging: 0.3374 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 5978.1 | uss_MB: 5210.6 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:16:08,922] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:16:08,943] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-42000.pth +rank0 [2026-10-28 16:16:09,116] INFO : Running validation on rank0 +rank0 [2026-10-28 16:16:09,116] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:16:09,116] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:16:10,409] INFO : saved batch0_epoch42000.parquet +rank0 [2026-10-28 16:16:10,553] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:16:10,684] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:16:10,758] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:16:10,829] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:16:10,898] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:16:10,967] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:16:11,036] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:16:11,152] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:16:11,282] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:16:11,414] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:16:11,485] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:16:11,555] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:16:11,624] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:16:11,693] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:16:11,762] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:16:11,864] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:16:11,870] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:16:11,873] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:16:11,874] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:16:11,876] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:16:11,879] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:16:11,881] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:16:11,883] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:16:11,884] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:16:11,886] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:16:11,889] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:16:11,890] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:16:11,892] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:16:11,895] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:16:11,896] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:16:11,898] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:16:19,475] INFO : VALIDATION | Step=42000/50000 | Train Loss=8.2783 | Valid Loss=8.0147 | Train Loss Unweighted=2.7126 | Valid Loss Unweighted=2.6181 | Stale=0 | ETA=57.9m +rank0 [2026-10-28 16:16:19,485] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:19,545] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:16:19,548] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:19,724] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:21,870] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:22,573] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:23,289] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:23,991] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:24,177] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:16:24,180] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:24,237] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:16:24,240] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:24,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:26,356] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:26,958] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:27,561] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:28,196] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:28,390] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:16:28,393] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:16:28,446] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:16:28,449] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:16:28,565] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:16:30,405] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:16:31,064] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:16:31,652] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:16:32,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:16:32,477] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:16:32,485] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:32,740] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:16:33,191] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:34,622] INFO : Plotted jets +rank0 [2026-10-28 16:16:35,738] INFO : Plotted jet ratio +rank0 [2026-10-28 16:16:35,771] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:35,998] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:16:36,374] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:38,334] INFO : Plotted jets +rank0 [2026-10-28 16:16:39,402] INFO : Plotted jet ratio +rank0 [2026-10-28 16:16:39,435] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 16:16:39,639] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_42000/clic_edm_qq_pf +rank0 [2026-10-28 16:16:40,056] INFO : Plotted number of elements +rank0 [2026-10-28 16:16:41,448] INFO : Plotted jets +rank0 [2026-10-28 16:16:42,557] INFO : Plotted jet ratio +rank0 [2026-10-28 16:17:23,707] INFO : Step 42100/50000 rank0 | Train Loss: 8.4585 | Train Loss Unweighted: 2.6235 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3224 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1271 | Regression_eta: 0.0112 | Regression_pt: 0.1251 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8197 | Classification_binary: 2.3224 | Regression_cos_phi: 0.6258 | Regression_energy: 1.7580 | Regression_eta: 0.5521 | Regression_pt: 1.8821 | Regression_sin_phi: 0.4985 | LR: 1.54e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7941s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1850 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.2317 | max_tensorboard_logging: 0.2950 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6164.5 | uss_MB: 5398.1 | child_rss_MB: 3979.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:17:23,919] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:04,197] INFO : Step 42200/50000 rank0 | Train Loss: 7.4983 | Train Loss Unweighted: 2.5543 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.2904 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1129 | Regression_eta: 0.0085 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7000 | Classification_binary: 2.2904 | Regression_cos_phi: 0.4133 | Regression_energy: 1.5609 | Regression_eta: 0.4188 | Regression_pt: 1.6872 | Regression_sin_phi: 0.4277 | LR: 1.53e-04 | DataLoad Time: 0.0017s | Model Forward Time: 1.6572s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0775 | max_loss: 0.0649 | max_backward: 0.2036 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.2931 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6164.5 | uss_MB: 5398.2 | child_rss_MB: 3979.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:18:04,411] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:18:10,653] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-10-28 16:18:10,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-10-28 16:18:44,430] INFO : Step 42300/50000 rank0 | Train Loss: 7.5322 | Train Loss Unweighted: 2.5345 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.2787 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1080 | Regression_eta: 0.0102 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7349 | Classification_binary: 2.2787 | Regression_cos_phi: 0.5315 | Regression_energy: 1.4937 | Regression_eta: 0.4997 | Regression_pt: 1.5769 | Regression_sin_phi: 0.4168 | LR: 1.52e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6926s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0861 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2074 | max_tensorboard_logging: 0.3095 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6203.6 | uss_MB: 5436.1 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:18:44,642] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:19:24,650] INFO : Step 42400/50000 rank0 | Train Loss: 7.6167 | Train Loss Unweighted: 2.5575 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.2967 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1086 | Regression_eta: 0.0081 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8257 | Classification_binary: 2.2967 | Regression_cos_phi: 0.4861 | Regression_energy: 1.5021 | Regression_eta: 0.3979 | Regression_pt: 1.6373 | Regression_sin_phi: 0.4707 | LR: 1.50e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.9127s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0818 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1468 | max_tensorboard_logging: 0.3952 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 6203.6 | uss_MB: 5436.1 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:19:24,864] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:04,739] INFO : Step 42500/50000 rank0 | Train Loss: 7.6569 | Train Loss Unweighted: 2.5782 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3123 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1143 | Regression_eta: 0.0080 | Regression_pt: 0.1099 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7830 | Classification_binary: 2.3123 | Regression_cos_phi: 0.5299 | Regression_energy: 1.5801 | Regression_eta: 0.3923 | Regression_pt: 1.6529 | Regression_sin_phi: 0.4063 | LR: 1.49e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.7544s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2821 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.3778 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6049.6 | uss_MB: 5280.3 | child_rss_MB: 3980.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:20:04,951] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:20:44,696] INFO : Step 42600/50000 rank0 | Train Loss: 7.1602 | Train Loss Unweighted: 2.4476 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.2110 | Regression_cos_phi: 0.0086 | Regression_energy: 0.0980 | Regression_eta: 0.0081 | Regression_pt: 0.0972 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6828 | Classification_binary: 2.2110 | Regression_cos_phi: 0.5718 | Regression_energy: 1.3551 | Regression_eta: 0.3982 | Regression_pt: 1.4630 | Regression_sin_phi: 0.4783 | LR: 1.48e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6943s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2087 | max_optimizer_step: 0.1413 | max_tensorboard_logging: 0.3945 | max_tensorboard_flush: 0.0119 | Memory: rss_MB: 6049.6 | uss_MB: 5280.6 | child_rss_MB: 3979.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:20:44,908] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:21:24,541] INFO : Step 42700/50000 rank0 | Train Loss: 7.9446 | Train Loss Unweighted: 2.5983 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3126 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1228 | Regression_eta: 0.0097 | Regression_pt: 0.1212 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.3126 | Regression_cos_phi: 0.5042 | Regression_energy: 1.6975 | Regression_eta: 0.4745 | Regression_pt: 1.8236 | Regression_sin_phi: 0.4010 | LR: 1.47e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7896s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0127 | max_forward: 0.0778 | max_loss: 0.0035 | max_backward: 0.3063 | max_optimizer_step: 0.1455 | max_tensorboard_logging: 0.3840 | max_tensorboard_flush: 0.0125 | Memory: rss_MB: 6049.6 | uss_MB: 5280.7 | child_rss_MB: 3976.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:21:24,754] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:04,726] INFO : Step 42800/50000 rank0 | Train Loss: 7.1961 | Train Loss Unweighted: 2.5301 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.2812 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1060 | Regression_eta: 0.0077 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7433 | Classification_binary: 2.2812 | Regression_cos_phi: 0.3818 | Regression_energy: 1.4661 | Regression_eta: 0.3774 | Regression_pt: 1.5873 | Regression_sin_phi: 0.3591 | LR: 1.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.7325s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1507 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2085 | max_tensorboard_logging: 0.2911 | max_tensorboard_flush: 0.0124 | Memory: rss_MB: 6049.6 | uss_MB: 5280.7 | child_rss_MB: 3977.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:22:04,938] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:22:45,226] INFO : Step 42900/50000 rank0 | Train Loss: 7.8925 | Train Loss Unweighted: 2.5584 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2783 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1190 | Regression_eta: 0.0082 | Regression_pt: 0.1185 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7054 | Classification_binary: 2.2783 | Regression_cos_phi: 0.5372 | Regression_energy: 1.6461 | Regression_eta: 0.4022 | Regression_pt: 1.7826 | Regression_sin_phi: 0.5406 | LR: 1.44e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.8347s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0017 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3489 | max_optimizer_step: 0.1412 | max_tensorboard_logging: 0.3545 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6049.6 | uss_MB: 5283.2 | child_rss_MB: 3980.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 78.9 +rank0 [2026-10-28 16:22:45,443] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:25,811] INFO : Step 43000/50000 rank0 | Train Loss: 8.3373 | Train Loss Unweighted: 2.5881 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.2820 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1319 | Regression_eta: 0.0102 | Regression_pt: 0.1288 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8064 | Classification_binary: 2.2820 | Regression_cos_phi: 0.5421 | Regression_energy: 1.8237 | Regression_eta: 0.5008 | Regression_pt: 1.9378 | Regression_sin_phi: 0.4444 | LR: 1.43e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8501s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0596 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3048 | max_optimizer_step: 0.1422 | max_tensorboard_logging: 0.4010 | max_tensorboard_flush: 0.0034 | Memory: rss_MB: 6049.6 | uss_MB: 5281.0 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:23:26,023] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:23:26,042] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-43000.pth +rank0 [2026-10-28 16:23:26,323] INFO : Running validation on rank0 +rank0 [2026-10-28 16:23:26,323] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:23:26,323] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:23:27,667] INFO : saved batch0_epoch43000.parquet +rank0 [2026-10-28 16:23:27,812] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:23:27,943] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:23:28,017] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:23:28,085] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:23:28,154] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:23:28,222] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:23:28,291] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:23:28,409] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:23:28,540] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:23:28,670] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:23:28,741] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:23:28,810] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:23:28,878] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:23:28,947] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:23:29,016] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:23:29,119] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:23:29,126] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:23:29,129] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:23:29,132] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:23:29,133] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:23:29,135] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:23:29,138] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:23:29,139] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:23:29,141] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:23:29,144] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:23:29,145] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:23:29,148] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:23:29,151] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:23:29,154] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:23:29,157] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:23:29,160] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:23:36,523] INFO : VALIDATION | Step=43000/50000 | Train Loss=8.3373 | Valid Loss=7.9925 | Train Loss Unweighted=2.5881 | Valid Loss Unweighted=2.6155 | Stale=0 | ETA=50.6m +rank0 [2026-10-28 16:23:36,533] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:36,620] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:23:36,624] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:36,807] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:38,717] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:39,426] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:40,141] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:40,853] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:41,044] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:23:41,047] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:41,116] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:23:41,119] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:41,225] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:43,126] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:43,730] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:44,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:44,955] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:45,157] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:23:45,160] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:23:45,216] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:23:45,219] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:23:45,335] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:23:47,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:23:47,692] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:23:48,374] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:23:49,040] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:23:49,199] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:23:49,207] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:49,465] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:23:49,915] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:51,339] INFO : Plotted jets +rank0 [2026-10-28 16:23:53,069] INFO : Plotted jet ratio +rank0 [2026-10-28 16:23:53,104] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:53,330] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:23:53,703] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:55,081] INFO : Plotted jets +rank0 [2026-10-28 16:23:56,147] INFO : Plotted jet ratio +rank0 [2026-10-28 16:23:56,182] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 16:23:56,388] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_43000/clic_edm_qq_pf +rank0 [2026-10-28 16:23:56,802] INFO : Plotted number of elements +rank0 [2026-10-28 16:23:58,187] INFO : Plotted jets +rank0 [2026-10-28 16:23:59,293] INFO : Plotted jet ratio +rank0 [2026-10-28 16:24:39,079] INFO : Step 43100/50000 rank0 | Train Loss: 8.0236 | Train Loss Unweighted: 2.5486 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.2615 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1212 | Regression_eta: 0.0108 | Regression_pt: 0.1208 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7994 | Classification_binary: 2.2615 | Regression_cos_phi: 0.5191 | Regression_energy: 1.6762 | Regression_eta: 0.5304 | Regression_pt: 1.8179 | Regression_sin_phi: 0.4191 | LR: 1.42e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7511s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0035 | max_backward: 0.2444 | max_optimizer_step: 0.1479 | max_tensorboard_logging: 0.3908 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3646.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:24:39,292] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:25:18,579] INFO : Step 43200/50000 rank0 | Train Loss: 8.0705 | Train Loss Unweighted: 2.6043 | Train Loss Components: Classification: 0.0196 | Classification_binary: 2.3090 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1276 | Regression_eta: 0.0075 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8223 | Classification_binary: 2.3090 | Regression_cos_phi: 0.5016 | Regression_energy: 1.7650 | Regression_eta: 0.3686 | Regression_pt: 1.8933 | Regression_sin_phi: 0.4108 | LR: 1.40e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8101s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1780 | max_loss: 0.0034 | max_backward: 0.2030 | max_optimizer_step: 0.2455 | max_tensorboard_logging: 0.2949 | max_tensorboard_flush: 0.0118 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3982.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:25:18,792] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:25:59,093] INFO : Step 43300/50000 rank0 | Train Loss: 7.9324 | Train Loss Unweighted: 2.5775 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2880 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1242 | Regression_eta: 0.0080 | Regression_pt: 0.1246 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7544 | Classification_binary: 2.2880 | Regression_cos_phi: 0.4443 | Regression_energy: 1.7178 | Regression_eta: 0.3907 | Regression_pt: 1.8741 | Regression_sin_phi: 0.4632 | LR: 1.39e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7861s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0035 | max_backward: 0.2186 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.4285 | max_tensorboard_flush: 0.0143 | Memory: rss_MB: 6240.6 | uss_MB: 5475.0 | child_rss_MB: 3984.1 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:25:59,307] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:26:39,097] INFO : Step 43400/50000 rank0 | Train Loss: 7.9300 | Train Loss Unweighted: 2.5945 | Train Loss Components: Classification: 0.0193 | Classification_binary: 2.3156 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1178 | Regression_eta: 0.0085 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8104 | Classification_binary: 2.3156 | Regression_cos_phi: 0.4990 | Regression_energy: 1.6297 | Regression_eta: 0.4179 | Regression_pt: 1.7613 | Regression_sin_phi: 0.4961 | LR: 1.38e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6722s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0829 | max_backward: 0.2031 | max_optimizer_step: 0.1441 | max_tensorboard_logging: 0.2881 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6240.6 | uss_MB: 5474.8 | child_rss_MB: 3981.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:26:39,311] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:18,958] INFO : Step 43500/50000 rank0 | Train Loss: 7.7281 | Train Loss Unweighted: 2.5779 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3026 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1176 | Regression_eta: 0.0082 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7908 | Classification_binary: 2.3026 | Regression_cos_phi: 0.4850 | Regression_energy: 1.6258 | Regression_eta: 0.4046 | Regression_pt: 1.7623 | Regression_sin_phi: 0.3569 | LR: 1.37e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0416 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.3105 | max_optimizer_step: 0.1463 | max_tensorboard_logging: 0.3846 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6122.0 | uss_MB: 5355.1 | child_rss_MB: 3980.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:27:19,172] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 47.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:27:58,734] INFO : Step 43600/50000 rank0 | Train Loss: 7.6677 | Train Loss Unweighted: 2.5158 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.2538 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1090 | Regression_eta: 0.0098 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7056 | Classification_binary: 2.2538 | Regression_cos_phi: 0.5962 | Regression_energy: 1.5070 | Regression_eta: 0.4807 | Regression_pt: 1.6400 | Regression_sin_phi: 0.4845 | LR: 1.36e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6691s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1004 | max_loss: 0.0035 | max_backward: 0.2029 | max_optimizer_step: 0.2030 | max_tensorboard_logging: 0.2880 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6122.0 | uss_MB: 5356.2 | child_rss_MB: 3978.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:27:58,946] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:28:39,484] INFO : Step 43700/50000 rank0 | Train Loss: 8.0854 | Train Loss Unweighted: 2.5878 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.3023 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1203 | Regression_eta: 0.0106 | Regression_pt: 0.1195 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7629 | Classification_binary: 2.3023 | Regression_cos_phi: 0.5257 | Regression_energy: 1.6642 | Regression_eta: 0.5192 | Regression_pt: 1.7982 | Regression_sin_phi: 0.5129 | LR: 1.35e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7767s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1791 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2320 | max_tensorboard_logging: 0.2943 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3981.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:28:39,699] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:18,813] INFO : Step 43800/50000 rank0 | Train Loss: 7.1666 | Train Loss Unweighted: 2.4593 | Train Loss Components: Classification: 0.0153 | Classification_binary: 2.2108 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1050 | Regression_eta: 0.0079 | Regression_pt: 0.1063 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6412 | Classification_binary: 2.2108 | Regression_cos_phi: 0.5084 | Regression_energy: 1.4520 | Regression_eta: 0.3880 | Regression_pt: 1.5986 | Regression_sin_phi: 0.3676 | LR: 1.34e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7179s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1528 | max_loss: 0.0035 | max_backward: 0.2030 | max_optimizer_step: 0.2037 | max_tensorboard_logging: 0.2900 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3983.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:29:19,025] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:29:59,377] INFO : Step 43900/50000 rank0 | Train Loss: 8.5324 | Train Loss Unweighted: 2.6790 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3767 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1281 | Regression_eta: 0.0096 | Regression_pt: 0.1264 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7551 | Classification_binary: 2.3767 | Regression_cos_phi: 0.6793 | Regression_energy: 1.7712 | Regression_eta: 0.4729 | Regression_pt: 1.9009 | Regression_sin_phi: 0.5763 | LR: 1.33e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7681s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3115 | max_optimizer_step: 0.1433 | max_tensorboard_logging: 0.3756 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3982.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:29:59,588] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:39,545] INFO : Step 44000/50000 rank0 | Train Loss: 7.8708 | Train Loss Unweighted: 2.6106 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3341 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1174 | Regression_eta: 0.0096 | Regression_pt: 0.1165 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.3341 | Regression_cos_phi: 0.5145 | Regression_energy: 1.6231 | Regression_eta: 0.4710 | Regression_pt: 1.7528 | Regression_sin_phi: 0.4198 | LR: 1.32e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.6680s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0012 | max_forward: 0.1804 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1421 | max_tensorboard_logging: 0.2802 | max_tensorboard_flush: 0.0098 | Memory: rss_MB: 6123.0 | uss_MB: 5356.3 | child_rss_MB: 3978.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:30:39,764] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:30:39,782] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-44000.pth +rank0 [2026-10-28 16:30:39,954] INFO : Running validation on rank0 +rank0 [2026-10-28 16:30:39,955] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:30:39,955] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:30:41,242] INFO : saved batch0_epoch44000.parquet +rank0 [2026-10-28 16:30:41,406] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:30:41,551] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:30:41,625] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:30:41,693] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:30:41,762] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:30:41,833] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:30:41,902] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:30:42,018] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:30:42,149] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:30:42,281] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:30:42,352] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:30:42,422] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:30:42,490] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:30:42,560] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:30:42,629] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:30:42,731] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:30:42,738] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:30:42,741] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:30:42,744] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:30:42,745] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:30:42,748] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:30:42,751] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:30:42,752] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:30:42,754] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:30:42,755] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:30:42,756] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:30:42,759] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:30:42,761] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:30:42,762] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:30:42,765] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:30:42,768] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:30:50,059] INFO : VALIDATION | Step=44000/50000 | Train Loss=7.8708 | Valid Loss=7.9560 | Train Loss Unweighted=2.6106 | Valid Loss Unweighted=2.6022 | Stale=0 | ETA=43.4m +rank0 [2026-10-28 16:30:50,069] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:50,132] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:30:50,135] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:30:50,362] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:30:52,348] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:30:52,981] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:30:53,619] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:30:54,288] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:30:54,474] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:30:54,477] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:54,534] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:30:54,536] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:30:54,655] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:30:56,704] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:30:57,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:30:57,921] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:30:58,528] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:30:58,717] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:30:58,721] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:30:58,774] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:30:58,777] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:30:58,904] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:31:00,905] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:31:01,479] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:31:02,145] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:31:02,706] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:31:02,895] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:31:02,903] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:31:03,156] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:31:03,608] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:05,035] INFO : Plotted jets +rank0 [2026-10-28 16:31:06,147] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:06,179] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:31:06,403] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:31:06,774] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:08,145] INFO : Plotted jets +rank0 [2026-10-28 16:31:09,837] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:09,869] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 16:31:10,073] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_44000/clic_edm_qq_pf +rank0 [2026-10-28 16:31:10,489] INFO : Plotted number of elements +rank0 [2026-10-28 16:31:11,872] INFO : Plotted jets +rank0 [2026-10-28 16:31:12,983] INFO : Plotted jet ratio +rank0 [2026-10-28 16:31:53,049] INFO : Step 44100/50000 rank0 | Train Loss: 8.0210 | Train Loss Unweighted: 2.6396 | Train Loss Components: Classification: 0.0187 | Classification_binary: 2.3585 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1190 | Regression_eta: 0.0084 | Regression_pt: 0.1181 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7840 | Classification_binary: 2.3585 | Regression_cos_phi: 0.5640 | Regression_energy: 1.6455 | Regression_eta: 0.4148 | Regression_pt: 1.7769 | Regression_sin_phi: 0.4773 | LR: 1.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6712s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0777 | max_loss: 0.0036 | max_backward: 0.2040 | max_optimizer_step: 0.1993 | max_tensorboard_logging: 0.2887 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3977.9 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:31:53,266] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:32:32,646] INFO : Step 44200/50000 rank0 | Train Loss: 7.2918 | Train Loss Unweighted: 2.4844 | Train Loss Components: Classification: 0.0178 | Classification_binary: 2.2436 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1005 | Regression_eta: 0.0102 | Regression_pt: 0.0968 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7455 | Classification_binary: 2.2436 | Regression_cos_phi: 0.5009 | Regression_energy: 1.3900 | Regression_eta: 0.4994 | Regression_pt: 1.4559 | Regression_sin_phi: 0.4565 | LR: 1.30e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8042s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0070 | max_forward: 0.0775 | max_loss: 0.0036 | max_backward: 0.3124 | max_optimizer_step: 0.1469 | max_tensorboard_logging: 0.3777 | max_tensorboard_flush: 0.0120 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3979.7 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:32:32,859] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:12,677] INFO : Step 44300/50000 rank0 | Train Loss: 8.4375 | Train Loss Unweighted: 2.6831 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.3849 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1266 | Regression_eta: 0.0117 | Regression_pt: 0.1239 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7685 | Classification_binary: 2.3849 | Regression_cos_phi: 0.5886 | Regression_energy: 1.7514 | Regression_eta: 0.5739 | Regression_pt: 1.8637 | Regression_sin_phi: 0.5066 | LR: 1.29e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8425s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0100 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.3042 | max_optimizer_step: 0.1499 | max_tensorboard_logging: 0.3834 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6314.7 | uss_MB: 5549.7 | child_rss_MB: 3980.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:33:12,889] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:33:53,471] INFO : Step 44400/50000 rank0 | Train Loss: 7.8882 | Train Loss Unweighted: 2.6080 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.3327 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1186 | Regression_eta: 0.0094 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8053 | Classification_binary: 2.3327 | Regression_cos_phi: 0.4659 | Regression_energy: 1.6399 | Regression_eta: 0.4628 | Regression_pt: 1.6920 | Regression_sin_phi: 0.4895 | LR: 1.28e-04 | DataLoad Time: 0.0018s | Model Forward Time: 2.0162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.1087 | max_forward: 0.1050 | max_loss: 0.0037 | max_backward: 0.3959 | max_optimizer_step: 0.1477 | max_tensorboard_logging: 0.3766 | max_tensorboard_flush: 0.0113 | Memory: rss_MB: 6314.7 | uss_MB: 5551.8 | child_rss_MB: 3985.0 | allocated_MB: 295.1 | reserved_MB: 3742.0 | inactive_split_MB: 94.9 +rank0 [2026-10-28 16:33:53,681] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:34:33,530] INFO : Step 44500/50000 rank0 | Train Loss: 7.0552 | Train Loss Unweighted: 2.5482 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.3085 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1023 | Regression_eta: 0.0069 | Regression_pt: 0.1011 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7387 | Classification_binary: 2.3085 | Regression_cos_phi: 0.4171 | Regression_energy: 1.4147 | Regression_eta: 0.3406 | Regression_pt: 1.5208 | Regression_sin_phi: 0.3148 | LR: 1.27e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6676s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0325 | max_backward: 0.2035 | max_optimizer_step: 0.1505 | max_tensorboard_logging: 0.3104 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6199.2 | uss_MB: 5433.2 | child_rss_MB: 3979.6 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:34:33,747] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:13,755] INFO : Step 44600/50000 rank0 | Train Loss: 7.6610 | Train Loss Unweighted: 2.5507 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.2764 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1171 | Regression_eta: 0.0067 | Regression_pt: 0.1177 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7502 | Classification_binary: 2.2764 | Regression_cos_phi: 0.4720 | Regression_energy: 1.6193 | Regression_eta: 0.3279 | Regression_pt: 1.7711 | Regression_sin_phi: 0.4442 | LR: 1.26e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7432s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1759 | max_loss: 0.0037 | max_backward: 0.2039 | max_optimizer_step: 0.1934 | max_tensorboard_logging: 0.2877 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6199.2 | uss_MB: 5434.7 | child_rss_MB: 3979.8 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:35:13,970] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:35:54,380] INFO : Step 44700/50000 rank0 | Train Loss: 7.9455 | Train Loss Unweighted: 2.5134 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2341 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1178 | Regression_eta: 0.0101 | Regression_pt: 0.1161 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7354 | Classification_binary: 2.2341 | Regression_cos_phi: 0.6096 | Regression_energy: 1.6289 | Regression_eta: 0.4983 | Regression_pt: 1.7462 | Regression_sin_phi: 0.4930 | LR: 1.25e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1375 | max_loss: 0.0036 | max_backward: 0.2031 | max_optimizer_step: 0.1950 | max_tensorboard_logging: 0.2778 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3977.2 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 94.6 +rank0 [2026-10-28 16:35:54,593] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:36:34,361] INFO : Step 44800/50000 rank0 | Train Loss: 7.5848 | Train Loss Unweighted: 2.5759 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3186 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1088 | Regression_eta: 0.0094 | Regression_pt: 0.1060 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7324 | Classification_binary: 2.3186 | Regression_cos_phi: 0.5660 | Regression_energy: 1.5043 | Regression_eta: 0.4608 | Regression_pt: 1.5950 | Regression_sin_phi: 0.4076 | LR: 1.24e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7565s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1844 | max_loss: 0.0036 | max_backward: 0.2034 | max_optimizer_step: 0.2368 | max_tensorboard_logging: 0.2784 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3976.4 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:36:34,575] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:14,499] INFO : Step 44900/50000 rank0 | Train Loss: 7.9453 | Train Loss Unweighted: 2.6254 | Train Loss Components: Classification: 0.0170 | Classification_binary: 2.3453 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1201 | Regression_eta: 0.0107 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7118 | Classification_binary: 2.3453 | Regression_cos_phi: 0.4835 | Regression_energy: 1.6612 | Regression_eta: 0.5281 | Regression_pt: 1.7603 | Regression_sin_phi: 0.4550 | LR: 1.23e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6042s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.2031 | max_optimizer_step: 0.2053 | max_tensorboard_logging: 0.2705 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6200.2 | uss_MB: 5434.9 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 96.6 +rank0 [2026-10-28 16:37:14,725] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 144.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:54,203] INFO : Step 45000/50000 rank0 | Train Loss: 7.6674 | Train Loss Unweighted: 2.6009 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.3346 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1121 | Regression_eta: 0.0070 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7013 | Classification_binary: 2.3346 | Regression_cos_phi: 0.5637 | Regression_energy: 1.5496 | Regression_eta: 0.3445 | Regression_pt: 1.7157 | Regression_sin_phi: 0.4581 | LR: 1.22e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7719s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0036 | max_backward: 0.3116 | max_optimizer_step: 0.1442 | max_tensorboard_logging: 0.3758 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6200.2 | uss_MB: 5435.0 | child_rss_MB: 3975.5 | allocated_MB: 291.4 | reserved_MB: 3742.0 | inactive_split_MB: 92.6 +rank0 [2026-10-28 16:37:54,418] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:37:54,494] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-45000.pth +rank0 [2026-10-28 16:37:54,675] INFO : Running validation on rank0 +rank0 [2026-10-28 16:37:54,675] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:37:54,675] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:37:55,952] INFO : saved batch0_epoch45000.parquet +rank0 [2026-10-28 16:37:56,098] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:37:56,229] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:37:56,302] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:37:56,370] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:37:56,438] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:37:56,506] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:37:56,575] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:37:56,691] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:37:56,821] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:37:56,952] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:37:57,023] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:37:57,091] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:37:57,158] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:37:57,228] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:37:57,297] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:37:57,398] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:37:57,404] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:37:57,412] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:37:57,420] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:37:57,422] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:37:57,425] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:37:57,426] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:37:57,429] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:37:57,429] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:37:57,432] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:37:57,435] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:37:57,436] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:37:57,439] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:37:57,440] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:37:57,442] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:37:57,443] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:38:04,733] INFO : VALIDATION | Step=45000/50000 | Train Loss=7.6674 | Valid Loss=7.9613 | Train Loss Unweighted=2.6009 | Valid Loss Unweighted=2.6039 | Stale=1 | ETA=36.2m +rank0 [2026-10-28 16:38:04,742] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:04,805] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:38:04,808] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:04,946] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:06,954] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:07,626] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:08,313] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:08,987] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:09,208] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:38:09,211] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:09,268] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:38:09,271] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:09,387] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:11,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:11,930] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:12,540] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:13,188] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:13,386] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:38:13,389] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:38:13,444] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:38:13,447] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:38:13,560] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:38:15,268] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:38:15,939] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:38:16,622] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:38:17,296] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:38:17,501] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:38:17,509] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:17,767] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:38:18,222] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:19,647] INFO : Plotted jets +rank0 [2026-10-28 16:38:20,751] INFO : Plotted jet ratio +rank0 [2026-10-28 16:38:20,783] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:21,010] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:38:21,387] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:22,755] INFO : Plotted jets +rank0 [2026-10-28 16:38:24,479] INFO : Plotted jet ratio +rank0 [2026-10-28 16:38:24,514] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 16:38:24,721] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_45000/clic_edm_qq_pf +rank0 [2026-10-28 16:38:25,133] INFO : Plotted number of elements +rank0 [2026-10-28 16:38:26,516] INFO : Plotted jets +rank0 [2026-10-28 16:38:27,712] INFO : Plotted jet ratio +rank0 [2026-10-28 16:39:08,198] INFO : Step 45100/50000 rank0 | Train Loss: 7.7868 | Train Loss Unweighted: 2.6053 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.3443 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1076 | Regression_eta: 0.0100 | Regression_pt: 0.1073 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8143 | Classification_binary: 2.3443 | Regression_cos_phi: 0.5720 | Regression_energy: 1.4877 | Regression_eta: 0.4924 | Regression_pt: 1.6149 | Regression_sin_phi: 0.4612 | LR: 1.21e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6604s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0864 | max_loss: 0.0036 | max_backward: 0.2041 | max_optimizer_step: 0.1957 | max_tensorboard_logging: 0.2793 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:39:08,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:39:48,199] INFO : Step 45200/50000 rank0 | Train Loss: 7.9533 | Train Loss Unweighted: 2.5811 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3033 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1157 | Regression_eta: 0.0091 | Regression_pt: 0.1172 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7806 | Classification_binary: 2.3033 | Regression_cos_phi: 0.5579 | Regression_energy: 1.5996 | Regression_eta: 0.4486 | Regression_pt: 1.7628 | Regression_sin_phi: 0.5005 | LR: 1.20e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7499s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.1690 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.1985 | max_tensorboard_logging: 0.2801 | max_tensorboard_flush: 0.0097 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3977.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:39:48,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:40:28,925] INFO : Step 45300/50000 rank0 | Train Loss: 7.6908 | Train Loss Unweighted: 2.5671 | Train Loss Components: Classification: 0.0172 | Classification_binary: 2.2890 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1208 | Regression_eta: 0.0080 | Regression_pt: 0.1186 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7186 | Classification_binary: 2.2890 | Regression_cos_phi: 0.4461 | Regression_energy: 1.6702 | Regression_eta: 0.3910 | Regression_pt: 1.7847 | Regression_sin_phi: 0.3913 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7136s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1320 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1980 | max_tensorboard_logging: 0.2888 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:40:29,140] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:08,535] INFO : Step 45400/50000 rank0 | Train Loss: 7.3959 | Train Loss Unweighted: 2.4640 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.2099 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1071 | Regression_eta: 0.0103 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6630 | Classification_binary: 2.2099 | Regression_cos_phi: 0.5201 | Regression_energy: 1.4816 | Regression_eta: 0.5054 | Regression_pt: 1.5876 | Regression_sin_phi: 0.4283 | LR: 1.19e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.8851s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0839 | max_forward: 0.0778 | max_loss: 0.0036 | max_backward: 0.3111 | max_optimizer_step: 0.1483 | max_tensorboard_logging: 0.3805 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6396.5 | uss_MB: 5629.4 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:41:08,750] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:41:48,497] INFO : Step 45500/50000 rank0 | Train Loss: 7.8399 | Train Loss Unweighted: 2.5047 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.2371 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1117 | Regression_eta: 0.0103 | Regression_pt: 0.1089 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7399 | Classification_binary: 2.2371 | Regression_cos_phi: 0.5699 | Regression_energy: 1.5441 | Regression_eta: 0.5059 | Regression_pt: 1.6379 | Regression_sin_phi: 0.6052 | LR: 1.18e-04 | DataLoad Time: 0.0033s | Model Forward Time: 1.8581s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0247 | max_forward: 0.0774 | max_loss: 0.0035 | max_backward: 0.3100 | max_optimizer_step: 0.1494 | max_tensorboard_logging: 0.4030 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6274.3 | uss_MB: 5507.1 | child_rss_MB: 3982.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:41:48,710] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:42:28,202] INFO : Step 45600/50000 rank0 | Train Loss: 7.7827 | Train Loss Unweighted: 2.5392 | Train Loss Components: Classification: 0.0190 | Classification_binary: 2.2732 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1117 | Regression_eta: 0.0116 | Regression_pt: 0.1080 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7965 | Classification_binary: 2.2732 | Regression_cos_phi: 0.5325 | Regression_energy: 1.5444 | Regression_eta: 0.5706 | Regression_pt: 1.6247 | Regression_sin_phi: 0.4409 | LR: 1.17e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7732s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1796 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2087 | max_tensorboard_logging: 0.2882 | max_tensorboard_flush: 0.0114 | Memory: rss_MB: 6274.3 | uss_MB: 5508.3 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:42:28,415] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:08,245] INFO : Step 45700/50000 rank0 | Train Loss: 8.4492 | Train Loss Unweighted: 2.6834 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.3851 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1277 | Regression_eta: 0.0131 | Regression_pt: 0.1224 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7577 | Classification_binary: 2.3851 | Regression_cos_phi: 0.5704 | Regression_energy: 1.7655 | Regression_eta: 0.6436 | Regression_pt: 1.8413 | Regression_sin_phi: 0.4856 | LR: 1.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7181s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1387 | max_loss: 0.0036 | max_backward: 0.2033 | max_optimizer_step: 0.2036 | max_tensorboard_logging: 0.2896 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6274.3 | uss_MB: 5508.6 | child_rss_MB: 3978.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:43:08,461] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 51.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:43:49,609] INFO : Step 45800/50000 rank0 | Train Loss: 7.2952 | Train Loss Unweighted: 2.4509 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.1959 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1088 | Regression_eta: 0.0075 | Regression_pt: 0.1069 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7350 | Classification_binary: 2.1959 | Regression_cos_phi: 0.4653 | Regression_energy: 1.5045 | Regression_eta: 0.3679 | Regression_pt: 1.6077 | Regression_sin_phi: 0.4190 | LR: 1.16e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8756s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0979 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.3094 | max_optimizer_step: 0.1484 | max_tensorboard_logging: 0.3767 | max_tensorboard_flush: 0.0117 | Memory: rss_MB: 6274.3 | uss_MB: 5508.7 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:43:49,824] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:44:29,916] INFO : Step 45900/50000 rank0 | Train Loss: 8.3838 | Train Loss Unweighted: 2.5962 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3052 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1213 | Regression_eta: 0.0130 | Regression_pt: 0.1194 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7311 | Classification_binary: 2.3052 | Regression_cos_phi: 0.6942 | Regression_energy: 1.6771 | Regression_eta: 0.6376 | Regression_pt: 1.7957 | Regression_sin_phi: 0.5429 | LR: 1.15e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7997s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0014 | max_forward: 0.1833 | max_loss: 0.0036 | max_backward: 0.2036 | max_optimizer_step: 0.2410 | max_tensorboard_logging: 0.3007 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6275.3 | uss_MB: 5508.9 | child_rss_MB: 3977.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:44:30,130] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:10,290] INFO : Step 46000/50000 rank0 | Train Loss: 7.9312 | Train Loss Unweighted: 2.6196 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.3430 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1171 | Regression_eta: 0.0097 | Regression_pt: 0.1159 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7335 | Classification_binary: 2.3430 | Regression_cos_phi: 0.5267 | Regression_energy: 1.6196 | Regression_eta: 0.4755 | Regression_pt: 1.7430 | Regression_sin_phi: 0.4899 | LR: 1.14e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7369s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0781 | max_loss: 0.0036 | max_backward: 0.2569 | max_optimizer_step: 0.1466 | max_tensorboard_logging: 0.3794 | max_tensorboard_flush: 0.0110 | Memory: rss_MB: 6275.3 | uss_MB: 5508.8 | child_rss_MB: 3978.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:45:10,506] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:45:10,542] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-46000.pth +rank0 [2026-10-28 16:45:10,722] INFO : Running validation on rank0 +rank0 [2026-10-28 16:45:10,722] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:45:10,722] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:45:12,080] INFO : saved batch0_epoch46000.parquet +rank0 [2026-10-28 16:45:12,224] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:45:12,355] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:45:12,429] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:45:12,497] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:45:12,565] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:45:12,634] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:45:12,704] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:45:12,820] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:45:12,951] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:45:13,082] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:45:13,153] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:45:13,222] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:45:13,290] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:45:13,359] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:45:13,430] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:45:13,530] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:45:13,537] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:45:13,539] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:45:13,540] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:45:13,543] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:45:13,544] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:45:13,547] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:45:13,549] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:45:13,550] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:45:13,553] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:45:13,556] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:45:13,557] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:45:13,559] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:45:13,562] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:45:13,570] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:45:13,579] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:45:21,043] INFO : VALIDATION | Step=46000/50000 | Train Loss=7.9312 | Valid Loss=7.9367 | Train Loss Unweighted=2.6196 | Valid Loss Unweighted=2.5952 | Stale=0 | ETA=28.9m +rank0 [2026-10-28 16:45:21,053] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:21,114] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:45:21,118] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:21,296] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:23,246] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:23,874] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:24,518] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:25,215] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:25,414] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:45:25,417] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:25,473] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:45:25,476] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:25,591] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:27,520] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:28,124] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:28,737] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:29,328] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:29,542] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 16:45:29,546] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:45:29,600] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:45:29,603] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:45:29,721] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:45:31,633] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:45:32,287] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:45:32,863] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:45:33,498] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:45:33,721] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:45:33,728] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:33,979] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:45:34,431] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:35,854] INFO : Plotted jets +rank0 [2026-10-28 16:45:36,972] INFO : Plotted jet ratio +rank0 [2026-10-28 16:45:37,005] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:37,229] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:45:37,601] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:39,549] INFO : Plotted jets +rank0 [2026-10-28 16:45:40,623] INFO : Plotted jet ratio +rank0 [2026-10-28 16:45:40,656] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 16:45:40,862] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_46000/clic_edm_qq_pf +rank0 [2026-10-28 16:45:41,277] INFO : Plotted number of elements +rank0 [2026-10-28 16:45:42,658] INFO : Plotted jets +rank0 [2026-10-28 16:45:43,762] INFO : Plotted jet ratio +rank0 [2026-10-28 16:46:24,822] INFO : Step 46100/50000 rank0 | Train Loss: 7.9594 | Train Loss Unweighted: 2.5880 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3016 | Regression_cos_phi: 0.0078 | Regression_energy: 0.1220 | Regression_eta: 0.0087 | Regression_pt: 0.1226 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7178 | Classification_binary: 2.3016 | Regression_cos_phi: 0.5185 | Regression_energy: 1.6874 | Regression_eta: 0.4255 | Regression_pt: 1.8449 | Regression_sin_phi: 0.4637 | LR: 1.13e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6421s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0307 | max_backward: 0.2032 | max_optimizer_step: 0.1485 | max_tensorboard_logging: 0.2947 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3979.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:46:25,037] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:47:04,762] INFO : Step 46200/50000 rank0 | Train Loss: 8.2248 | Train Loss Unweighted: 2.5759 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.2739 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1317 | Regression_eta: 0.0087 | Regression_pt: 0.1267 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7693 | Classification_binary: 2.2739 | Regression_cos_phi: 0.5840 | Regression_energy: 1.8211 | Regression_eta: 0.4271 | Regression_pt: 1.9068 | Regression_sin_phi: 0.4426 | LR: 1.13e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2567 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3900 | max_tensorboard_flush: 0.0130 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3981.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:47:04,977] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:47:45,355] INFO : Step 46300/50000 rank0 | Train Loss: 7.9768 | Train Loss Unweighted: 2.4779 | Train Loss Components: Classification: 0.0182 | Classification_binary: 2.1829 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1262 | Regression_eta: 0.0081 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7637 | Classification_binary: 2.1829 | Regression_cos_phi: 0.5947 | Regression_energy: 1.7458 | Regression_eta: 0.3989 | Regression_pt: 1.9077 | Regression_sin_phi: 0.3830 | LR: 1.12e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0246 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3111 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3942 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6472.1 | uss_MB: 5707.8 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 78.6 +rank0 [2026-10-28 16:47:45,569] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:48:26,227] INFO : Step 46400/50000 rank0 | Train Loss: 7.2676 | Train Loss Unweighted: 2.4940 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.2479 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1026 | Regression_eta: 0.0085 | Regression_pt: 0.1030 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7299 | Classification_binary: 2.2479 | Regression_cos_phi: 0.4916 | Regression_energy: 1.4193 | Regression_eta: 0.4173 | Regression_pt: 1.5500 | Regression_sin_phi: 0.4116 | LR: 1.11e-04 | DataLoad Time: 0.0025s | Model Forward Time: 1.8336s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1049 | max_loss: 0.0036 | max_backward: 0.3224 | max_optimizer_step: 0.1512 | max_tensorboard_logging: 0.3393 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6472.1 | uss_MB: 5709.9 | child_rss_MB: 3979.9 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:48:26,444] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:06,247] INFO : Step 46500/50000 rank0 | Train Loss: 7.5536 | Train Loss Unweighted: 2.5266 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.2638 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1124 | Regression_eta: 0.0076 | Regression_pt: 0.1090 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7895 | Classification_binary: 2.2638 | Regression_cos_phi: 0.5309 | Regression_energy: 1.5546 | Regression_eta: 0.3717 | Regression_pt: 1.6398 | Regression_sin_phi: 0.4033 | LR: 1.11e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8548s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.4059 | max_tensorboard_flush: 0.0121 | Memory: rss_MB: 6344.0 | uss_MB: 5579.1 | child_rss_MB: 3980.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:49:06,462] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:49:46,304] INFO : Step 46600/50000 rank0 | Train Loss: 7.9133 | Train Loss Unweighted: 2.5919 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3106 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1198 | Regression_eta: 0.0086 | Regression_pt: 0.1193 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7153 | Classification_binary: 2.3106 | Regression_cos_phi: 0.5287 | Regression_energy: 1.6570 | Regression_eta: 0.4207 | Regression_pt: 1.7953 | Regression_sin_phi: 0.4856 | LR: 1.10e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.6344s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0095 | max_backward: 0.2033 | max_optimizer_step: 0.1520 | max_tensorboard_logging: 0.2995 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6346.0 | uss_MB: 5580.0 | child_rss_MB: 3978.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:49:46,517] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:50:26,440] INFO : Step 46700/50000 rank0 | Train Loss: 8.1561 | Train Loss Unweighted: 2.5758 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2807 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1244 | Regression_eta: 0.0098 | Regression_pt: 0.1262 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7734 | Classification_binary: 2.2807 | Regression_cos_phi: 0.5378 | Regression_energy: 1.7200 | Regression_eta: 0.4815 | Regression_pt: 1.8985 | Regression_sin_phi: 0.4642 | LR: 1.10e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7497s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2378 | max_optimizer_step: 0.1515 | max_tensorboard_logging: 0.3889 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6346.0 | uss_MB: 5580.3 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:50:26,653] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:06,936] INFO : Step 46800/50000 rank0 | Train Loss: 7.7938 | Train Loss Unweighted: 2.4785 | Train Loss Components: Classification: 0.0175 | Classification_binary: 2.1889 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1274 | Regression_eta: 0.0080 | Regression_pt: 0.1227 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7351 | Classification_binary: 2.1889 | Regression_cos_phi: 0.5098 | Regression_energy: 1.7623 | Regression_eta: 0.3948 | Regression_pt: 1.8453 | Regression_sin_phi: 0.3575 | LR: 1.09e-04 | DataLoad Time: 0.0020s | Model Forward Time: 2.0185s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0018 | max_forward: 0.2084 | max_loss: 0.0036 | max_backward: 0.3967 | max_optimizer_step: 0.1446 | max_tensorboard_logging: 0.3993 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 6346.0 | uss_MB: 5583.0 | child_rss_MB: 3982.3 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 80.9 +rank0 [2026-10-28 16:51:07,150] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:51:47,507] INFO : Step 46900/50000 rank0 | Train Loss: 7.7673 | Train Loss Unweighted: 2.6570 | Train Loss Components: Classification: 0.0179 | Classification_binary: 2.3826 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1188 | Regression_eta: 0.0075 | Regression_pt: 0.1162 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7485 | Classification_binary: 2.3826 | Regression_cos_phi: 0.4940 | Regression_energy: 1.6427 | Regression_eta: 0.3694 | Regression_pt: 1.7478 | Regression_sin_phi: 0.3823 | LR: 1.09e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7716s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2662 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3785 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6346.0 | uss_MB: 5581.0 | child_rss_MB: 3980.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:51:47,721] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:28,504] INFO : Step 47000/50000 rank0 | Train Loss: 7.5421 | Train Loss Unweighted: 2.5632 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.2940 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1158 | Regression_eta: 0.0085 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7788 | Classification_binary: 2.2940 | Regression_cos_phi: 0.3614 | Regression_energy: 1.6018 | Regression_eta: 0.4153 | Regression_pt: 1.7214 | Regression_sin_phi: 0.3693 | LR: 1.08e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7032s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1003 | max_loss: 0.0036 | max_backward: 0.2042 | max_optimizer_step: 0.1997 | max_tensorboard_logging: 0.3030 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6346.0 | uss_MB: 5581.0 | child_rss_MB: 3978.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:52:28,717] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:52:28,736] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-47000.pth +rank0 [2026-10-28 16:52:28,910] INFO : Running validation on rank0 +rank0 [2026-10-28 16:52:28,910] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:52:28,910] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:52:30,197] INFO : saved batch0_epoch47000.parquet +rank0 [2026-10-28 16:52:30,342] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:52:30,472] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:52:30,546] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:52:30,615] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:52:30,685] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:52:30,754] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:52:30,823] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:52:30,937] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:52:31,068] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:52:31,199] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:52:31,271] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:52:31,339] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:52:31,409] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:52:31,478] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:52:31,547] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:52:31,649] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:52:31,655] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:52:31,657] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:52:31,660] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:52:31,661] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:52:31,664] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:52:31,667] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:52:31,667] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:52:31,670] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:52:31,671] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:52:31,673] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:52:31,674] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:52:31,677] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:52:31,679] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:52:31,680] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:52:31,683] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:52:39,056] INFO : VALIDATION | Step=47000/50000 | Train Loss=7.5421 | Valid Loss=7.9361 | Train Loss Unweighted=2.5632 | Valid Loss Unweighted=2.5935 | Stale=0 | ETA=21.7m +rank0 [2026-10-28 16:52:39,071] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:39,150] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:52:39,154] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:39,358] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:41,499] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:42,218] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:42,915] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:43,592] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:43,777] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:43,780] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:43,836] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:52:43,839] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:43,966] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:45,945] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:46,553] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:47,201] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:47,799] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:47,997] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:48,000] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:52:48,055] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 16:52:48,058] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 16:52:48,183] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:52:50,178] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 16:52:50,803] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 16:52:51,388] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 16:52:52,026] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 16:52:52,225] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:52:52,232] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:52,485] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ttbar_pf +rank0 [2026-10-28 16:52:52,938] INFO : Plotted number of elements +rank0 [2026-10-28 16:52:54,363] INFO : Plotted jets +rank0 [2026-10-28 16:52:55,422] INFO : Plotted jet ratio +rank0 [2026-10-28 16:52:55,457] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:55,680] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:52:56,057] INFO : Plotted number of elements +rank0 [2026-10-28 16:52:57,429] INFO : Plotted jets +rank0 [2026-10-28 16:52:59,120] INFO : Plotted jet ratio +rank0 [2026-10-28 16:52:59,153] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 16:52:59,357] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_47000/clic_edm_qq_pf +rank0 [2026-10-28 16:52:59,770] INFO : Plotted number of elements +rank0 [2026-10-28 16:53:01,152] INFO : Plotted jets +rank0 [2026-10-28 16:53:02,254] INFO : Plotted jet ratio +rank0 [2026-10-28 16:53:42,020] INFO : Step 47100/50000 rank0 | Train Loss: 7.6987 | Train Loss Unweighted: 2.4662 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.1796 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1250 | Regression_eta: 0.0078 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7068 | Classification_binary: 2.1796 | Regression_cos_phi: 0.4648 | Regression_energy: 1.7289 | Regression_eta: 0.3845 | Regression_pt: 1.8557 | Regression_sin_phi: 0.3786 | LR: 1.07e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0777 | max_loss: 0.0828 | max_backward: 0.2038 | max_optimizer_step: 0.1513 | max_tensorboard_logging: 0.2850 | max_tensorboard_flush: 0.0101 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3978.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:53:42,233] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:54:22,575] INFO : Step 47200/50000 rank0 | Train Loss: 7.8956 | Train Loss Unweighted: 2.5648 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2881 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1170 | Regression_eta: 0.0089 | Regression_pt: 0.1157 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7555 | Classification_binary: 2.2881 | Regression_cos_phi: 0.5667 | Regression_energy: 1.6178 | Regression_eta: 0.4392 | Regression_pt: 1.7403 | Regression_sin_phi: 0.4879 | LR: 1.07e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8531s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0289 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3118 | max_optimizer_step: 0.1504 | max_tensorboard_logging: 0.3826 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 16:54:22,789] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:55:02,771] INFO : Step 47300/50000 rank0 | Train Loss: 7.6050 | Train Loss Unweighted: 2.5608 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3006 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1091 | Regression_eta: 0.0083 | Regression_pt: 0.1083 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7690 | Classification_binary: 2.3006 | Regression_cos_phi: 0.4950 | Regression_energy: 1.5093 | Regression_eta: 0.4067 | Regression_pt: 1.6296 | Regression_sin_phi: 0.4949 | LR: 1.06e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7291s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2289 | max_optimizer_step: 0.1497 | max_tensorboard_logging: 0.3789 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3980.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:55:02,985] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:55:43,505] INFO : Step 47400/50000 rank0 | Train Loss: 8.0405 | Train Loss Unweighted: 2.5912 | Train Loss Components: Classification: 0.0184 | Classification_binary: 2.3006 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1243 | Regression_eta: 0.0088 | Regression_pt: 0.1235 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7718 | Classification_binary: 2.3006 | Regression_cos_phi: 0.4731 | Regression_energy: 1.7186 | Regression_eta: 0.4325 | Regression_pt: 1.8584 | Regression_sin_phi: 0.4855 | LR: 1.06e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7225s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0038 | max_backward: 0.2038 | max_optimizer_step: 0.1534 | max_tensorboard_logging: 0.3740 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6554.1 | uss_MB: 5791.2 | child_rss_MB: 3977.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:55:43,719] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:56:24,077] INFO : Step 47500/50000 rank0 | Train Loss: 8.2188 | Train Loss Unweighted: 2.7061 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.4081 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1291 | Regression_eta: 0.0094 | Regression_pt: 0.1270 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7432 | Classification_binary: 2.4081 | Regression_cos_phi: 0.4736 | Regression_energy: 1.7859 | Regression_eta: 0.4607 | Regression_pt: 1.9103 | Regression_sin_phi: 0.4371 | LR: 1.06e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8547s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0149 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.3098 | max_optimizer_step: 0.1524 | max_tensorboard_logging: 0.3992 | max_tensorboard_flush: 0.0112 | Memory: rss_MB: 6419.3 | uss_MB: 5656.0 | child_rss_MB: 3981.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:56:24,293] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:04,280] INFO : Step 47600/50000 rank0 | Train Loss: 6.9840 | Train Loss Unweighted: 2.3953 | Train Loss Components: Classification: 0.0167 | Classification_binary: 2.1559 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1012 | Regression_eta: 0.0074 | Regression_pt: 0.1002 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7014 | Classification_binary: 2.1559 | Regression_cos_phi: 0.4680 | Regression_energy: 1.3995 | Regression_eta: 0.3641 | Regression_pt: 1.5078 | Regression_sin_phi: 0.3873 | LR: 1.05e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8293s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0012 | max_forward: 0.0782 | max_loss: 0.0036 | max_backward: 0.3079 | max_optimizer_step: 0.1516 | max_tensorboard_logging: 0.3856 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6420.3 | uss_MB: 5657.0 | child_rss_MB: 3974.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 16:57:04,494] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:57:43,936] INFO : Step 47700/50000 rank0 | Train Loss: 8.5214 | Train Loss Unweighted: 2.6744 | Train Loss Components: Classification: 0.0177 | Classification_binary: 2.3670 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1320 | Regression_eta: 0.0100 | Regression_pt: 0.1290 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7396 | Classification_binary: 2.3670 | Regression_cos_phi: 0.6070 | Regression_energy: 1.8249 | Regression_eta: 0.4900 | Regression_pt: 1.9412 | Regression_sin_phi: 0.5518 | LR: 1.05e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.7508s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0013 | max_forward: 0.1424 | max_loss: 0.0035 | max_backward: 0.2032 | max_optimizer_step: 0.1890 | max_tensorboard_logging: 0.2891 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6420.3 | uss_MB: 5657.3 | child_rss_MB: 3975.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:57:44,151] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:58:23,662] INFO : Step 47800/50000 rank0 | Train Loss: 7.9297 | Train Loss Unweighted: 2.6392 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.3603 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1200 | Regression_eta: 0.0088 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7862 | Classification_binary: 2.3603 | Regression_cos_phi: 0.4835 | Regression_energy: 1.6596 | Regression_eta: 0.4327 | Regression_pt: 1.7449 | Regression_sin_phi: 0.4626 | LR: 1.04e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.6902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0014 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.2126 | max_optimizer_step: 0.1500 | max_tensorboard_logging: 0.3670 | max_tensorboard_flush: 0.0096 | Memory: rss_MB: 6421.3 | uss_MB: 5657.3 | child_rss_MB: 3980.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 16:58:23,875] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:03,601] INFO : Step 47900/50000 rank0 | Train Loss: 8.0374 | Train Loss Unweighted: 2.5968 | Train Loss Components: Classification: 0.0191 | Classification_binary: 2.3178 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1161 | Regression_eta: 0.0100 | Regression_pt: 0.1163 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8021 | Classification_binary: 2.3178 | Regression_cos_phi: 0.5358 | Regression_energy: 1.6060 | Regression_eta: 0.4905 | Regression_pt: 1.7497 | Regression_sin_phi: 0.5355 | LR: 1.04e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7273s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1535 | max_loss: 0.0035 | max_backward: 0.2033 | max_optimizer_step: 0.1972 | max_tensorboard_logging: 0.2854 | max_tensorboard_flush: 0.0099 | Memory: rss_MB: 6421.3 | uss_MB: 5657.3 | child_rss_MB: 3979.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 16:59:03,816] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:43,299] INFO : Step 48000/50000 rank0 | Train Loss: 7.5096 | Train Loss Unweighted: 2.5823 | Train Loss Components: Classification: 0.0186 | Classification_binary: 2.3281 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1073 | Regression_eta: 0.0081 | Regression_pt: 0.1050 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7800 | Classification_binary: 2.3281 | Regression_cos_phi: 0.5154 | Regression_energy: 1.4843 | Regression_eta: 0.3974 | Regression_pt: 1.5798 | Regression_sin_phi: 0.4247 | LR: 1.04e-04 | DataLoad Time: 0.0019s | Model Forward Time: 2.0061s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0938 | max_forward: 0.1053 | max_loss: 0.0038 | max_backward: 0.3941 | max_optimizer_step: 0.1508 | max_tensorboard_logging: 0.3753 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6422.3 | uss_MB: 5660.2 | child_rss_MB: 3977.0 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 16:59:43,514] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 16:59:43,590] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-48000.pth +rank0 [2026-10-28 16:59:43,765] INFO : Running validation on rank0 +rank0 [2026-10-28 16:59:43,766] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 16:59:43,766] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 16:59:45,077] INFO : saved batch0_epoch48000.parquet +rank0 [2026-10-28 16:59:45,221] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 16:59:45,351] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 16:59:45,424] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 16:59:45,492] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 16:59:45,561] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 16:59:45,629] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 16:59:45,698] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 16:59:45,814] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 16:59:45,945] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 16:59:46,077] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 16:59:46,148] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 16:59:46,218] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 16:59:46,286] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 16:59:46,357] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 16:59:46,427] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 16:59:46,531] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 16:59:46,538] INFO : plotted pt_target histogram +rank0 [2026-10-28 16:59:46,540] INFO : plotted pt_pred histogram +rank0 [2026-10-28 16:59:46,543] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 16:59:46,544] INFO : plotted eta_target histogram +rank0 [2026-10-28 16:59:46,547] INFO : plotted eta_pred histogram +rank0 [2026-10-28 16:59:46,548] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 16:59:46,550] INFO : plotted sphi_target histogram +rank0 [2026-10-28 16:59:46,551] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 16:59:46,554] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 16:59:46,556] INFO : plotted cphi_target histogram +rank0 [2026-10-28 16:59:46,557] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 16:59:46,560] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 16:59:46,562] INFO : plotted energy_target histogram +rank0 [2026-10-28 16:59:46,563] INFO : plotted energy_pred histogram +rank0 [2026-10-28 16:59:46,566] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 16:59:54,040] INFO : VALIDATION | Step=48000/50000 | Train Loss=7.5096 | Valid Loss=7.9276 | Train Loss Unweighted=2.5823 | Valid Loss Unweighted=2.5875 | Stale=0 | ETA=14.5m +rank0 [2026-10-28 16:59:54,050] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:59:54,112] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 16:59:54,115] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 16:59:54,300] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 16:59:56,264] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 16:59:56,890] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 16:59:57,558] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 16:59:58,267] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 16:59:58,485] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 16:59:58,488] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 16:59:58,546] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 16:59:58,549] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 16:59:58,662] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:00:00,557] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:00:01,163] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:00:01,796] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:00:02,396] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:00:02,604] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:00:02,607] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:00:02,662] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:00:02,665] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:00:02,789] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:00:04,767] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:00:05,383] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:00:05,964] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:00:06,614] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:00:06,825] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:00:06,833] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:00:07,087] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:00:07,541] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:08,983] INFO : Plotted jets +rank0 [2026-10-28 17:00:10,110] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:10,142] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:00:10,367] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:00:10,745] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:12,134] INFO : Plotted jets +rank0 [2026-10-28 17:00:13,880] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:13,922] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 17:00:14,131] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_48000/clic_edm_qq_pf +rank0 [2026-10-28 17:00:14,546] INFO : Plotted number of elements +rank0 [2026-10-28 17:00:15,948] INFO : Plotted jets +rank0 [2026-10-28 17:00:17,064] INFO : Plotted jet ratio +rank0 [2026-10-28 17:00:57,143] INFO : Step 48100/50000 rank0 | Train Loss: 7.6490 | Train Loss Unweighted: 2.6035 | Train Loss Components: Classification: 0.0164 | Classification_binary: 2.3436 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1097 | Regression_eta: 0.0088 | Regression_pt: 0.1082 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6856 | Classification_binary: 2.3436 | Regression_cos_phi: 0.5542 | Regression_energy: 1.5170 | Regression_eta: 0.4340 | Regression_pt: 1.6279 | Regression_sin_phi: 0.4868 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.6679s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0013 | max_forward: 0.0783 | max_loss: 0.0038 | max_backward: 0.2034 | max_optimizer_step: 0.1875 | max_tensorboard_logging: 0.2824 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:00:57,356] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:01:37,199] INFO : Step 48200/50000 rank0 | Train Loss: 7.4357 | Train Loss Unweighted: 2.5648 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3011 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1144 | Regression_eta: 0.0069 | Regression_pt: 0.1121 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7174 | Classification_binary: 2.3011 | Regression_cos_phi: 0.4042 | Regression_energy: 1.5826 | Regression_eta: 0.3413 | Regression_pt: 1.6869 | Regression_sin_phi: 0.4021 | LR: 1.03e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1635 | max_loss: 0.0037 | max_backward: 0.2036 | max_optimizer_step: 0.1868 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3975.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:01:37,413] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:02:16,950] INFO : Step 48300/50000 rank0 | Train Loss: 7.5699 | Train Loss Unweighted: 2.5425 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.2805 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1100 | Regression_eta: 0.0089 | Regression_pt: 0.1101 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7771 | Classification_binary: 2.2805 | Regression_cos_phi: 0.4607 | Regression_energy: 1.5205 | Regression_eta: 0.4362 | Regression_pt: 1.6559 | Regression_sin_phi: 0.4390 | LR: 1.03e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8345s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0772 | max_loss: 0.0034 | max_backward: 0.3045 | max_optimizer_step: 0.1502 | max_tensorboard_logging: 0.3799 | max_tensorboard_flush: 0.0273 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3977.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:02:17,161] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 49.0°C 136.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 42.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:02:56,854] INFO : Step 48400/50000 rank0 | Train Loss: 7.7636 | Train Loss Unweighted: 2.5522 | Train Loss Components: Classification: 0.0199 | Classification_binary: 2.2848 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1112 | Regression_eta: 0.0094 | Regression_pt: 0.1115 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8338 | Classification_binary: 2.2848 | Regression_cos_phi: 0.5945 | Regression_energy: 1.5371 | Regression_eta: 0.4625 | Regression_pt: 1.6773 | Regression_sin_phi: 0.3737 | LR: 1.02e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.8544s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0046 | max_forward: 0.0779 | max_loss: 0.0036 | max_backward: 0.3113 | max_optimizer_step: 0.1545 | max_tensorboard_logging: 0.3882 | max_tensorboard_flush: 0.0105 | Memory: rss_MB: 6620.1 | uss_MB: 5857.1 | child_rss_MB: 3981.5 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 86.6 +rank0 [2026-10-28 17:02:57,068] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:03:36,592] INFO : Step 48500/50000 rank0 | Train Loss: 7.5133 | Train Loss Unweighted: 2.5305 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2774 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1060 | Regression_eta: 0.0077 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7144 | Classification_binary: 2.2774 | Regression_cos_phi: 0.5903 | Regression_energy: 1.4656 | Regression_eta: 0.3772 | Regression_pt: 1.5733 | Regression_sin_phi: 0.5152 | LR: 1.02e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8419s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1828 | max_loss: 0.0034 | max_backward: 0.2032 | max_optimizer_step: 0.2491 | max_tensorboard_logging: 0.2833 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6620.1 | uss_MB: 5857.2 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:03:36,807] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 45.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:16,631] INFO : Step 48600/50000 rank0 | Train Loss: 8.4022 | Train Loss Unweighted: 2.5995 | Train Loss Components: Classification: 0.0189 | Classification_binary: 2.3159 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1159 | Regression_eta: 0.0185 | Regression_pt: 0.1127 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7931 | Classification_binary: 2.3159 | Regression_cos_phi: 0.5717 | Regression_energy: 1.6033 | Regression_eta: 0.9092 | Regression_pt: 1.6955 | Regression_sin_phi: 0.5137 | LR: 1.02e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.7759s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.1846 | max_loss: 0.0036 | max_backward: 0.2036 | max_optimizer_step: 0.2019 | max_tensorboard_logging: 0.2844 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6492.1 | uss_MB: 5729.3 | child_rss_MB: 3979.0 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:04:16,848] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:04:56,487] INFO : Step 48700/50000 rank0 | Train Loss: 7.9583 | Train Loss Unweighted: 2.5666 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.2963 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1117 | Regression_eta: 0.0107 | Regression_pt: 0.1105 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8177 | Classification_binary: 2.2963 | Regression_cos_phi: 0.6340 | Regression_energy: 1.5441 | Regression_eta: 0.5265 | Regression_pt: 1.6626 | Regression_sin_phi: 0.4772 | LR: 1.02e-04 | DataLoad Time: 0.0022s | Model Forward Time: 1.8356s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0017 | max_forward: 0.1043 | max_loss: 0.1093 | max_backward: 0.2876 | max_optimizer_step: 0.1484 | max_tensorboard_logging: 0.2852 | max_tensorboard_flush: 0.0109 | Memory: rss_MB: 6492.1 | uss_MB: 5731.4 | child_rss_MB: 3978.5 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:04:56,702] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 43.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:05:36,414] INFO : Step 48800/50000 rank0 | Train Loss: 7.8042 | Train Loss Unweighted: 2.5026 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.2147 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1257 | Regression_eta: 0.0064 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7148 | Classification_binary: 2.2147 | Regression_cos_phi: 0.5287 | Regression_energy: 1.7378 | Regression_eta: 0.3162 | Regression_pt: 1.8532 | Regression_sin_phi: 0.4389 | LR: 1.01e-04 | DataLoad Time: 0.0026s | Model Forward Time: 1.7694s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0780 | max_loss: 0.0036 | max_backward: 0.2812 | max_optimizer_step: 0.1515 | max_tensorboard_logging: 0.3705 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6492.1 | uss_MB: 5729.8 | child_rss_MB: 3977.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:05:36,630] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 43.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:16,659] INFO : Step 48900/50000 rank0 | Train Loss: 7.5086 | Train Loss Unweighted: 2.4769 | Train Loss Components: Classification: 0.0166 | Classification_binary: 2.2133 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1119 | Regression_eta: 0.0092 | Regression_pt: 0.1108 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6968 | Classification_binary: 2.2133 | Regression_cos_phi: 0.4922 | Regression_energy: 1.5474 | Regression_eta: 0.4512 | Regression_pt: 1.6671 | Regression_sin_phi: 0.4407 | LR: 1.01e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7554s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.0773 | max_loss: 0.0034 | max_backward: 0.2478 | max_optimizer_step: 0.1465 | max_tensorboard_logging: 0.3748 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6492.1 | uss_MB: 5730.2 | child_rss_MB: 3975.7 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 84.6 +rank0 [2026-10-28 17:06:16,873] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:56,281] INFO : Step 49000/50000 rank0 | Train Loss: 7.4955 | Train Loss Unweighted: 2.5158 | Train Loss Components: Classification: 0.0169 | Classification_binary: 2.2432 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1186 | Regression_eta: 0.0075 | Regression_pt: 0.1171 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7082 | Classification_binary: 2.2432 | Regression_cos_phi: 0.4272 | Regression_energy: 1.6407 | Regression_eta: 0.3672 | Regression_pt: 1.7621 | Regression_sin_phi: 0.3470 | LR: 1.01e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.7595s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0012 | max_forward: 0.1836 | max_loss: 0.0037 | max_backward: 0.2037 | max_optimizer_step: 0.2164 | max_tensorboard_logging: 0.2751 | max_tensorboard_flush: 0.0104 | Memory: rss_MB: 6492.1 | uss_MB: 5730.2 | child_rss_MB: 3977.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:06:56,495] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 140.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:06:56,513] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-49000.pth +rank0 [2026-10-28 17:06:56,686] INFO : Running validation on rank0 +rank0 [2026-10-28 17:06:56,687] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:06:56,687] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:06:57,986] INFO : saved batch0_epoch49000.parquet +rank0 [2026-10-28 17:06:58,133] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:06:58,265] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:06:58,339] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:06:58,407] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:06:58,476] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:06:58,546] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:06:58,614] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:06:58,729] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:06:58,860] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:06:58,991] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:06:59,062] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:06:59,132] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:06:59,200] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:06:59,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:06:59,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:06:59,437] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:06:59,443] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:06:59,446] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:06:59,448] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:06:59,449] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:06:59,450] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:06:59,453] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:06:59,456] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:06:59,456] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:06:59,459] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:06:59,461] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:06:59,462] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:06:59,465] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:06:59,467] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:06:59,468] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:06:59,471] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:07:07,036] INFO : VALIDATION | Step=49000/50000 | Train Loss=7.4955 | Valid Loss=7.9046 | Train Loss Unweighted=2.5158 | Valid Loss Unweighted=2.5833 | Stale=0 | ETA=7.2m +rank0 [2026-10-28 17:07:07,046] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:07,108] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:07:07,111] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:07,297] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:09,156] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:09,791] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:10,427] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:11,047] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:11,231] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:07:11,235] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:11,291] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:07:11,294] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:11,420] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:13,359] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:14,003] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:14,710] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:15,311] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:15,513] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:07:15,516] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:07:15,571] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:07:15,574] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:07:15,698] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:07:17,634] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:07:18,240] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:07:18,817] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:07:19,443] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:07:19,655] INFO : Time taken to make predictions on device 0 is: 0.06 min +rank0 [2026-10-28 17:07:19,663] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:19,920] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:07:20,373] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:21,799] INFO : Plotted jets +rank0 [2026-10-28 17:07:22,910] INFO : Plotted jet ratio +rank0 [2026-10-28 17:07:22,942] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:23,167] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:07:23,541] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:25,549] INFO : Plotted jets +rank0 [2026-10-28 17:07:26,608] INFO : Plotted jet ratio +rank0 [2026-10-28 17:07:26,641] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 17:07:26,846] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_49000/clic_edm_qq_pf +rank0 [2026-10-28 17:07:27,263] INFO : Plotted number of elements +rank0 [2026-10-28 17:07:28,642] INFO : Plotted jets +rank0 [2026-10-28 17:07:29,743] INFO : Plotted jet ratio +rank0 [2026-10-28 17:08:10,776] INFO : Step 49100/50000 rank0 | Train Loss: 8.2601 | Train Loss Unweighted: 2.6619 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.3761 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1203 | Regression_eta: 0.0117 | Regression_pt: 0.1175 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7552 | Classification_binary: 2.3761 | Regression_cos_phi: 0.5764 | Regression_energy: 1.6637 | Regression_eta: 0.5773 | Regression_pt: 1.7675 | Regression_sin_phi: 0.5438 | LR: 1.01e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8564s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0106 | max_forward: 0.0781 | max_loss: 0.0037 | max_backward: 0.3106 | max_optimizer_step: 0.1572 | max_tensorboard_logging: 0.3786 | max_tensorboard_flush: 0.0100 | Memory: rss_MB: 6680.6 | uss_MB: 5920.6 | child_rss_MB: 3982.1 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:08:10,990] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 137.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 90.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:08:51,508] INFO : Step 49200/50000 rank0 | Train Loss: 8.2600 | Train Loss Unweighted: 2.5716 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2782 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1244 | Regression_eta: 0.0099 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7540 | Classification_binary: 2.2782 | Regression_cos_phi: 0.6665 | Regression_energy: 1.7202 | Regression_eta: 0.4880 | Regression_pt: 1.8361 | Regression_sin_phi: 0.5170 | LR: 1.01e-04 | DataLoad Time: 0.0023s | Model Forward Time: 1.9572s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.1046 | max_forward: 0.0777 | max_loss: 0.0037 | max_backward: 0.3116 | max_optimizer_step: 0.1565 | max_tensorboard_logging: 0.3801 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6680.6 | uss_MB: 5920.6 | child_rss_MB: 3980.4 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:08:51,724] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 135.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:09:31,249] INFO : Step 49300/50000 rank0 | Train Loss: 7.5964 | Train Loss Unweighted: 2.5544 | Train Loss Components: Classification: 0.0183 | Classification_binary: 2.2882 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1126 | Regression_eta: 0.0083 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7665 | Classification_binary: 2.2882 | Regression_cos_phi: 0.3899 | Regression_energy: 1.5572 | Regression_eta: 0.4076 | Regression_pt: 1.6919 | Regression_sin_phi: 0.4951 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0785 | max_loss: 0.0037 | max_backward: 0.2033 | max_optimizer_step: 0.1881 | max_tensorboard_logging: 0.2846 | max_tensorboard_flush: 0.0106 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3978.9 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:09:31,464] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:10:11,268] INFO : Step 49400/50000 rank0 | Train Loss: 7.3948 | Train Loss Unweighted: 2.5036 | Train Loss Components: Classification: 0.0176 | Classification_binary: 2.2500 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1057 | Regression_eta: 0.0081 | Regression_pt: 0.1071 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7368 | Classification_binary: 2.2500 | Regression_cos_phi: 0.4954 | Regression_energy: 1.4616 | Regression_eta: 0.3979 | Regression_pt: 1.6106 | Regression_sin_phi: 0.4426 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.6834s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.0842 | max_loss: 0.0037 | max_backward: 0.2035 | max_optimizer_step: 0.1907 | max_tensorboard_logging: 0.2892 | max_tensorboard_flush: 0.0108 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3976.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:10:11,480] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 142.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:10:51,330] INFO : Step 49500/50000 rank0 | Train Loss: 7.5640 | Train Loss Unweighted: 2.4239 | Train Loss Components: Classification: 0.0168 | Classification_binary: 2.1548 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1153 | Regression_eta: 0.0100 | Regression_pt: 0.1119 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7058 | Classification_binary: 2.1548 | Regression_cos_phi: 0.5089 | Regression_energy: 1.5942 | Regression_eta: 0.4921 | Regression_pt: 1.6839 | Regression_sin_phi: 0.4243 | LR: 1.00e-04 | DataLoad Time: 0.0021s | Model Forward Time: 1.7111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0776 | max_loss: 0.0037 | max_backward: 0.2043 | max_optimizer_step: 0.1809 | max_tensorboard_logging: 0.2926 | max_tensorboard_flush: 0.0115 | Memory: rss_MB: 6680.6 | uss_MB: 5920.7 | child_rss_MB: 3976.6 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:10:51,542] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:11:31,775] INFO : Step 49600/50000 rank0 | Train Loss: 7.7997 | Train Loss Unweighted: 2.5712 | Train Loss Components: Classification: 0.0165 | Classification_binary: 2.2874 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1256 | Regression_eta: 0.0090 | Regression_pt: 0.1188 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6921 | Classification_binary: 2.2874 | Regression_cos_phi: 0.4427 | Regression_energy: 1.7368 | Regression_eta: 0.4407 | Regression_pt: 1.7866 | Regression_sin_phi: 0.4134 | LR: 1.00e-04 | DataLoad Time: 0.0028s | Model Forward Time: 1.7940s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0017 | max_forward: 0.1054 | max_loss: 0.0039 | max_backward: 0.3198 | max_optimizer_step: 0.1519 | max_tensorboard_logging: 0.3003 | max_tensorboard_flush: 0.0111 | Memory: rss_MB: 6561.0 | uss_MB: 5803.9 | child_rss_MB: 3980.6 | allocated_MB: 295.1 | reserved_MB: 3790.0 | inactive_split_MB: 82.9 +rank0 [2026-10-28 17:11:31,988] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 143.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:11,844] INFO : Step 49700/50000 rank0 | Train Loss: 7.8176 | Train Loss Unweighted: 2.5364 | Train Loss Components: Classification: 0.0180 | Classification_binary: 2.2590 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1192 | Regression_eta: 0.0085 | Regression_pt: 0.1158 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7522 | Classification_binary: 2.2590 | Regression_cos_phi: 0.5899 | Regression_energy: 1.6480 | Regression_eta: 0.4194 | Regression_pt: 1.7417 | Regression_sin_phi: 0.4074 | LR: 1.00e-04 | DataLoad Time: 0.0019s | Model Forward Time: 1.6790s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0013 | max_forward: 0.0782 | max_loss: 0.0570 | max_backward: 0.2035 | max_optimizer_step: 0.1537 | max_tensorboard_logging: 0.2818 | max_tensorboard_flush: 0.0103 | Memory: rss_MB: 6561.0 | uss_MB: 5801.9 | child_rss_MB: 3978.8 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:12:12,058] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 141.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 42.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 41.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 44.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:12:51,718] INFO : Step 49800/50000 rank0 | Train Loss: 8.1509 | Train Loss Unweighted: 2.5941 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.3054 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1208 | Regression_eta: 0.0105 | Regression_pt: 0.1210 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.8510 | Classification_binary: 2.3054 | Regression_cos_phi: 0.5099 | Regression_energy: 1.6710 | Regression_eta: 0.5168 | Regression_pt: 1.8209 | Regression_sin_phi: 0.4760 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8536s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0301 | max_forward: 0.0774 | max_loss: 0.0034 | max_backward: 0.3099 | max_optimizer_step: 0.1518 | max_tensorboard_logging: 0.3800 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6561.0 | uss_MB: 5802.2 | child_rss_MB: 3976.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 80.6 +rank0 [2026-10-28 17:12:51,932] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 93.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:13:32,802] INFO : Step 49900/50000 rank0 | Train Loss: 8.0252 | Train Loss Unweighted: 2.6481 | Train Loss Components: Classification: 0.0171 | Classification_binary: 2.3611 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1236 | Regression_eta: 0.0092 | Regression_pt: 0.1215 | Regression_sin_phi: 0.0079 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.7154 | Classification_binary: 2.3611 | Regression_cos_phi: 0.5045 | Regression_energy: 1.7097 | Regression_eta: 0.4535 | Regression_pt: 1.8282 | Regression_sin_phi: 0.4528 | LR: 1.00e-04 | DataLoad Time: 0.0018s | Model Forward Time: 1.8208s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0038 | max_forward: 0.0784 | max_loss: 0.0036 | max_backward: 0.3090 | max_optimizer_step: 0.1530 | max_tensorboard_logging: 0.3776 | max_tensorboard_flush: 0.0107 | Memory: rss_MB: 6561.0 | uss_MB: 5802.2 | child_rss_MB: 3978.3 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:13:33,016] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 138.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 46.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 44.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 45.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 46.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:13,250] INFO : Step 50000/50000 rank0 | Train Loss: 7.4666 | Train Loss Unweighted: 2.4566 | Train Loss Components: Classification: 0.0158 | Classification_binary: 2.1871 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1153 | Regression_eta: 0.0071 | Regression_pt: 0.1164 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 41.8959 | Classification_binary: 1.0000 | Regression_cos_phi: 66.7309 | Regression_energy: 13.8286 | Regression_eta: 49.1345 | Regression_pt: 15.0444 | Regression_sin_phi: 57.0666 | Task Weighted Losses: Classification: 0.6603 | Classification_binary: 2.1871 | Regression_cos_phi: 0.5033 | Regression_energy: 1.5945 | Regression_eta: 0.3488 | Regression_pt: 1.7513 | Regression_sin_phi: 0.4214 | LR: 1.00e-04 | DataLoad Time: 0.0020s | Model Forward Time: 1.8184s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0013 | max_forward: 0.1839 | max_loss: 0.0035 | max_backward: 0.2039 | max_optimizer_step: 0.2479 | max_tensorboard_logging: 0.2853 | max_tensorboard_flush: 0.0102 | Memory: rss_MB: 6561.0 | uss_MB: 5802.3 | child_rss_MB: 3980.2 | allocated_MB: 291.4 | reserved_MB: 3790.0 | inactive_split_MB: 82.6 +rank0 [2026-10-28 17:14:13,462] INFO : + +========================================== ROCm System Management Interface ========================================== +==================================================== Concise Info ==================================================== +Device Node IDs Temp Power Partitions SCLK MCLK Fan Perf PwrCap VRAM% GPU% + (DID, GUID) (Edge) (Avg) (Mem, Compute, ID)  +====================================================================================================================== +0 4 0x7408, 63582 50.0°C 139.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 7% 0% +1 5 0x7408, 51740 47.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +2 6 0x7408, 15961 44.0°C 89.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +3 7 0x7408, 3099 43.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +4 8 0x7408, 13395 43.0°C 94.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +5 9 0x7408, 1553 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +6 10 0x7408, 62036 45.0°C 92.0W N/A, N/A, 0 800Mhz 1600Mhz 0% manual 500.0W 0% 0% +7 11 0x7408, 49174 44.0°C N/A N/A, N/A, 0 800Mhz 1600Mhz 0% manual 0.0W 0% 0% +====================================================================================================================== +================================================ End of ROCm SMI Log ================================================= + +rank0 [2026-10-28 17:14:13,480] INFO : saving checkpoint /pfs/lustrep2/scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/checkpoints/checkpoint-50000.pth +rank0 [2026-10-28 17:14:13,648] INFO : Running validation on rank0 +rank0 [2026-10-28 17:14:13,649] INFO : ResumableSampler physical:valid resetting +rank0 [2026-10-28 17:14:13,649] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-10-28 17:14:14,956] INFO : saved batch0_epoch50000.parquet +rank0 [2026-10-28 17:14:15,099] INFO : plotted energy_elemtype1 +rank0 [2026-10-28 17:14:15,231] INFO : plotted pt_elemtype1 +rank0 [2026-10-28 17:14:15,305] INFO : plotted energy_elemtype1_corr +rank0 [2026-10-28 17:14:15,373] INFO : plotted pt_elemtype1_corr +rank0 [2026-10-28 17:14:15,442] INFO : plotted eta_elemtype1_corr +rank0 [2026-10-28 17:14:15,511] INFO : plotted sphi_elemtype1_corr +rank0 [2026-10-28 17:14:15,580] INFO : plotted cphi_elemtype1_corr +rank0 [2026-10-28 17:14:15,695] INFO : plotted sig_proba_elemtype1 +rank0 [2026-10-28 17:14:15,826] INFO : plotted energy_elemtype2 +rank0 [2026-10-28 17:14:15,958] INFO : plotted pt_elemtype2 +rank0 [2026-10-28 17:14:16,030] INFO : plotted energy_elemtype2_corr +rank0 [2026-10-28 17:14:16,100] INFO : plotted pt_elemtype2_corr +rank0 [2026-10-28 17:14:16,168] INFO : plotted eta_elemtype2_corr +rank0 [2026-10-28 17:14:16,237] INFO : plotted sphi_elemtype2_corr +rank0 [2026-10-28 17:14:16,306] INFO : plotted cphi_elemtype2_corr +rank0 [2026-10-28 17:14:16,406] INFO : plotted sig_proba_elemtype2 +rank0 [2026-10-28 17:14:16,412] INFO : plotted pt_target histogram +rank0 [2026-10-28 17:14:16,415] INFO : plotted pt_pred histogram +rank0 [2026-10-28 17:14:16,416] INFO : plotted pt_ratio histogram +rank0 [2026-10-28 17:14:16,419] INFO : plotted eta_target histogram +rank0 [2026-10-28 17:14:16,420] INFO : plotted eta_pred histogram +rank0 [2026-10-28 17:14:16,422] INFO : plotted eta_ratio histogram +rank0 [2026-10-28 17:14:16,425] INFO : plotted sphi_target histogram +rank0 [2026-10-28 17:14:16,426] INFO : plotted sphi_pred histogram +rank0 [2026-10-28 17:14:16,429] INFO : plotted sphi_ratio histogram +rank0 [2026-10-28 17:14:16,431] INFO : plotted cphi_target histogram +rank0 [2026-10-28 17:14:16,432] INFO : plotted cphi_pred histogram +rank0 [2026-10-28 17:14:16,435] INFO : plotted cphi_ratio histogram +rank0 [2026-10-28 17:14:16,437] INFO : plotted energy_target histogram +rank0 [2026-10-28 17:14:16,440] INFO : plotted energy_pred histogram +rank0 [2026-10-28 17:14:16,441] INFO : plotted energy_ratio histogram +rank0 [2026-10-28 17:14:24,047] INFO : VALIDATION | Step=50000/50000 | Train Loss=7.4666 | Valid Loss=7.8930 | Train Loss Unweighted=2.4566 | Valid Loss Unweighted=2.5849 | Stale=0 | ETA=0.0m +rank0 [2026-10-28 17:14:24,057] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:24,118] INFO : test_dataset: clic_edm_ttbar_pf, 510 +rank0 [2026-10-28 17:14:24,121] INFO : Running predictions on clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:24,304] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:26,364] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:27,038] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:27,729] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:28,401] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:28,576] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:28,579] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:28,636] INFO : test_dataset: clic_edm_ww_fullhad_pf, 510 +rank0 [2026-10-28 17:14:28,639] INFO : Running predictions on clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:28,760] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:30,787] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:31,390] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:32,041] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:32,639] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:32,849] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:32,852] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-10-28 17:14:32,906] INFO : test_dataset: clic_edm_qq_pf, 510 +rank0 [2026-10-28 17:14:32,909] INFO : Running predictions on clic_edm_qq_pf +rank0 [2026-10-28 17:14:33,029] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-10-28 17:14:35,054] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_0.parquet +rank0 [2026-10-28 17:14:35,640] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_1.parquet +rank0 [2026-10-28 17:14:36,222] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_2.parquet +rank0 [2026-10-28 17:14:36,858] INFO : Saved predictions at /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf/pred_0_3.parquet +rank0 [2026-10-28 17:14:37,070] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-10-28 17:14:37,078] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:37,329] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ttbar_pf +rank0 [2026-10-28 17:14:37,780] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:39,202] INFO : Plotted jets +rank0 [2026-10-28 17:14:40,320] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:40,351] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:40,572] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_ww_fullhad_pf +rank0 [2026-10-28 17:14:40,947] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:42,319] INFO : Plotted jets +rank0 [2026-10-28 17:14:44,000] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:44,032] INFO : Loading data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 17:14:44,232] INFO : Loaded data for plotting from /scratch/project_465001293/patajoos/particleflow/experiments/clic_cld_backbone_benchmark/clic_pf_heptv2_seed12345_20261028_111202_849554/preds_step_50000/clic_edm_qq_pf +rank0 [2026-10-28 17:14:44,645] INFO : Plotted number of elements +rank0 [2026-10-28 17:14:46,031] INFO : Plotted jets +rank0 [2026-10-28 17:14:47,128] INFO : Plotted jet ratio +rank0 [2026-10-28 17:14:47,737] INFO : Training completed. Total time on device 0: 362.238min diff --git a/clic/clusters/v3.2.0/comparison_metrics/provenance.json b/clic/clusters/v3.2.0/comparison_metrics/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd93dc97277643919f983f16a0633812762afa4 --- /dev/null +++ b/clic/clusters/v3.2.0/comparison_metrics/provenance.json @@ -0,0 +1,24 @@ +{ + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "study": "20261030_clic_cld_checkpoint_summary", + "selected_step": 50000, + "sources": "Original per-run history JSON and train TensorBoard streams, retained alongside these summaries.", + "metrics": { + "f1": "Count-aggregated validation particle F1. Capped matching cost 2.0, acceptance deltaR<0.1 and relative pT error<0.5, no PID requirement.", + "pt_mae": "Mean absolute relative pT error among accepted pairs; conditional and truncated by acceptance, not inclusive resolution.", + "jet_iqr": "Equal mean of ttbar, WW fullhad, qq IQRs of angular-matched predicted/target jet pT ratios.", + "jet_recall": "Equal-process response-qualified jet recall, with abs(pT ratio - 1)<0.5.", + "met": "Full-validation mean absolute error of MET magnitude in GeV.", + "p3": "Detailed 512-entry subset mean norm of event three-momentum difference, GeV.", + "steady_s": "Median seconds/step across adjacent 100-step train TensorBoard intervals, excluding validation boundaries, warmup and session changes.", + "compute_s": "Median instrumented rank-zero forward+loss+backward+optimizer time, seconds.", + "peak_gib": "Maximum recorded allocator peak across ranks and sessions, per logical device. May include prior validation allocations." + }, + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ] +} diff --git a/clic/clusters/v3.2.0/comparison_metrics/study_methodology.md b/clic/clusters/v3.2.0/comparison_metrics/study_methodology.md new file mode 100644 index 0000000000000000000000000000000000000000..d40607cfea1c2d71542e91632ef20b56a9c172a0 --- /dev/null +++ b/clic/clusters/v3.2.0/comparison_metrics/study_methodology.md @@ -0,0 +1,207 @@ +# CLIC and CLD checkpoint summary, 30 October 2026 + +Twenty summary slides compare the eight runs in the 28 October +`clic_cld_backbone_benchmark` campaign. The common endpoint is **50,000 optimizer +steps**. Evolution curves receive priority over endpoint-only comparisons. +This is a snapshot of local results, not a report of current remote job status. + +## Deliverables + +- [PDF slides](output/checkpoint_summary.slides.pdf) +- [Self-contained HTML slides](output/checkpoint_summary.slides.html), arrow-key navigation +- [Source notebook](checkpoint_summary.ipynb) and [executed notebook](output/checkpoint_summary.executed.ipynb) +- [Numerical scorecard](output/scorecard.csv), [summary](output/summary.json), [resource series](output/resources.json) +- `output/slides.json`: generated slide copy, figure paths and source categories +- `output/*.svg` and `output/*.png`: standalone scientific figures + +The figures are generated from archived data by `study_analysis.py`. Tables and +slide copy are HTML text. No external CDN, mutable experiment path, or earlier +study archive is needed to render. This deliverable follows the notebook/HTML/PDF +format in the parent README; it does not include a PowerPoint conversion. + +## Findings + +HEPTv2 improves final jet IQR, MET error and detailed event momentum error in +all four detector/representation pairs. On CLIC hits, F1 rises from 0.7747 to +0.8623 (+8.76 percentage points) and the equal-process jet IQR decreases from +0.1579 to 0.0995 (37.0%). On CLD hits, F1 rises from 0.8458 to 0.8624 +(+1.67 points) and jet IQR decreases from 0.1243 to 0.1040 (16.4%). + +Hit HEPTv2 has higher final particle F1 than track/cluster HEPTv2 in both +detectors. Track/cluster HEPTv2 retains lower conditional particle momentum +error, jet IQR, MET error and event momentum-vector error. These statements +refer to the recorded metrics and configured target populations, not paired +statistical significance or full inclusive momentum resolution. + +The median ordinary training-interval estimates are about 0.11 s/step for PF +attention, 0.40–0.41 s for PF HEPTv2, 2.36–2.38 s for hit attention and +2.84–2.89 s for hit HEPTv2. PF uses one logical accelerator device; hits use +eight. At 50k steps this projects to roughly 55–58 times the device cost for +hit HEPTv2 versus PF HEPTv2, excluding validation, queueing and input +preprocessing. This is not an inference or end-to-end reconstruction benchmark. +Exposed rank-zero batch wait has medians near 2 ms for PF and 17 ms for hits. +No storage byte counters, CPU utilization or bandwidth measurements support a +broader I/O-bottleneck conclusion. + +## Runs and configuration + +All runs use seed 12345, revision +`6b80b983477321cf708e5043fb1d2ec15dd398bc`, global batch 128, 50k scheduled steps, +LAMB, BF16, LR 0.001 with cosine decay, and no compilation. All completed +histories end at 50k. Run IDs record launch preparation on 28 October 2026 at +11:12; hit continuations were submitted on 29 October as recorded in their +manifests. Full log timestamps and exact commands remain archived. + +| Archive name | Exact run ID | Devices | History points | Resume checkpoint | +| --- | --- | ---: | ---: | --- | +| cld_hits_attention | `cld_hits_attention_seed12345_20261028_111201_979187` | 8 | 50 | 35000 | +| cld_hits_heptv2 | `cld_hits_heptv2_seed12345_20261028_111202_875252` | 8 | 49 | 30000 | +| cld_pf_attention | `cld_pf_attention_seed12345_20261028_111201_967206` | 1 | 50 | none | +| cld_pf_heptv2 | `cld_pf_heptv2_seed12345_20261028_111202_024020` | 1 | 50 | none | +| clic_hits_attention | `clic_hits_attention_seed12345_20261028_111202_121851` | 8 | 50 | 35000 | +| clic_hits_heptv2 | `clic_hits_heptv2_seed12345_20261028_111202_012645` | 8 | 50 | 29000 | +| clic_pf_attention | `clic_pf_attention_seed12345_20261028_111202_052283` | 1 | 50 | none | +| clic_pf_heptv2 | `clic_pf_heptv2_seed12345_20261028_111202_849554` | 1 | 50 | none | + +Hardware provenance comes from `scenario-manifest.json`: platform +`lumi_mi250x`, eight logical devices available per platform/node. Actual model +world size is one for track/cluster and eight for hits, verified against resolved +configuration and logs. The generic `gpu_type: l40` in train-config is a stale +recipe default, not evidence that these jobs ran on L40 hardware. Device-hours +count logical MI250X accelerator devices/GCDs, not whole dual-GCD packages. + +All models use width 256, six shared backbone layers. PF attention/HEPTv2 have +3,114,258 / 8,639,954 parameters. Hit attention/HEPTv2 have +7,260,244 / 12,786,964. The hit decoder uses four global-Flash layers and 256 +slots, supervised grouping, tracker-neighborhood features and dedicated tracker +momentum readout. HEPTv2 uses deterministic hashing, block size 128 and activation +checkpointing. These are fixed recipe comparisons, not equal-parameter or +pure-backbone-kernel benchmarks. Losses/objectives and readouts differ between +PF and hit models, so weighted loss is not used as cross-representation evidence. + +Both detectors use ttbar, WW fullhad and qq. Dataset names are +`{clic,cld}_edm_{ttbar,ww_fullhad,qq}_{pf,hits}`, with all ten numbered configs. +PF TFDS is 3.3.0 and hit TFDS is 3.2.2. Exact production/data locations are +preserved only as provenance in original configuration files. Rendering never +reads those locations. At 50k each run has about 2.37 training epochs / 6.4M +training-event presentations, not 6.4M unique events. + +## Metrics and sampling + +Particle curves use history `valid/metrics/particle/*` scalars. The archived +validation matcher minimizes `(deltaR/0.1)^2 + (log(pT ratio)/log(2))^2`, capped +at 2 per pair. Accepted matches require deltaR < 0.1 and relative pT error < 0.5, +without PID agreement. F1/efficiency aggregate counts through the validation +accumulator, rather than averaging process-level F1. Conditional momentum MAE +is mean absolute relative pT error among accepted pairs; the 50% acceptance cut +truncates its tails. It is not a central-68% width or inclusive resolution. + +`nvalid=512` with ten split configs selects 51 entries per config, 510 per +process, 1,530 total. The distributed sampler pads the hit validation collection +to 1,536 entries. Detailed momentum metrics apply to four batches per rank, +512 entries globally for both representations, but rank partitioning changes +which events are in this subset. `momentum/all/event/p3_error_mean` is the mean +norm of the difference between summed predicted and target three-momenta, +not the mean of individual particle momentum errors. Basic MET and energy +metrics use the full validation collection. MET error compares magnitudes. + +Jet curves use top-level `step//jet_ratio/jet_ratio_target_to_pred_pt/*` +scalars from the test/plot cycle. Ratios are predicted / target pT. IQR is +Q75−Q25 among angular-matched jets, with no additional response acceptance cut. +Response-qualified recall counts angular matches satisfying |ratio−1| < 0.5 +relative to all target jets. Each displayed curve is an equal mean of the three +process-specific summaries, not an IQR of pooled jets. `ntest=512` likewise +requests 51 events from each of ten configs per process before distributed +padding; sample identity and duplicate entries were not independently audited. + +PF and hit target means differ slightly (e.g. CLD 70.565 versus 70.655), which +can include both target-definition and sampler effects. We have not verified +exact truth-event identity across representations, nor rescored predictions +with a common event subset. Backbone comparisons within a representation have +the same nominal configuration and validation counts, but have no paired-event +confidence intervals or training-seed replication. Repeated validation/test +inspection makes the comparisons exploratory. Earlier studies have different +data and matching defaults, and their metric values are not mixed into this +campaign. CLD hit HEPTv2 lacks `step_30000.json`; plots connect 29k to 31k without +synthesizing a 30k observation. + +## Resource method + +`extract_resources.py` reads every archived train event file separately with +unbounded scalar retention. It retains the latest wall-time observation for +replayed steps, preserving stream identity to reject cross-session intervals. +All required scalar series have aligned step sets. There are 500 observations +per model at steps 100–50,000 after deduplication. + +Ordinary throughput uses elapsed wall time between adjacent 100-step TensorBoard +observations from the same event file, excluding intervals beginning at a +multiple of 1,000 (which cross validation/test work), intervals before 1,000, +and nonadjacent/restart boundaries. The median seconds per optimizer step is +robust to isolated stalls but is an estimate, not total campaign elapsed time. +Each 100-step interval still contains one diagnostic step. Projected 50k device +cost multiplies this median by 50,000 and by actual world size. It excludes +validation, test plotting, checkpoint cycles at excluded boundaries, scheduling +downtime, startup and abandoned/replayed training work. It does not include PF +tracking/clustering preprocessing. + +Instrumented-step curves and phase breakdowns use rank-zero raw phase timers, +not sums of maxima across different ranks. Compute is forward + loss + backward ++ optimizer. The enclosing `time_model_forward` includes training, backward and +logging, so it is not used as pure forward. `step/time_train_total` includes +synchronized diagnostics and memory logging. Its median can be much greater +than the ordinary interval estimate. The remaining phase is total minus listed +phases; negative residuals are rejected. It includes diagnostic and collective +work not separately plotted. + +Loader wait is the exposed wall time in `next(train_iterator)` on rank zero. +Transfer is the synchronized batch-to-device phase. Neither measures storage +service time, byte throughput, asynchronous worker effort or the slowest rank's +I/O. Resource plots use all raw sampled observations; thick timing lines are +centered 11-sample medians. Input multiplicities quoted on slide 15 are medians +of the per-step, across-rank batch-average element counts. + +Allocator memory is the maximum across ranks of historical peak allocation at +step end, and can include earlier validation allocations. Restart resets it. +This differs from reserved/cached memory and total VRAM occupancy. CPU RSS is +the mean training-process footprint across ranks; it omits child RSS and is not +unique node memory. MB tag values use MiB, divided by 1024 for GiB. The compact +resource JSON also retains child RSS, USS, current reserved memory and padding +for later inspection. No inference benchmark is inferred from these logs. + +**Timer trap:** history `step_valid_time` is calculated as current time minus +session start minus the last training-step duration. It is cumulative and +resets on resume, not a validation-cycle duration. Neither it nor a sum of +history `step_total_time` values is used as validation or total training cost. +The archived revision's `training.py` documents this behavior. + +## Archive and reproduction + +Run from the repository root: + +```bash +bash notebooks/studies/20261030_clic_cld_checkpoint_summary/render.sh +``` + +The script executes the source notebook, which re-extracts resources, regenerates +figures/tables/copy, and emits each slide. It writes the executed notebook and +self-contained HTML, then headless Chrome generates PDF and Poppler renders page +PNGs. Override `PYTHON_BIN` / `CHROME_BIN` if needed. Dependencies are nbformat, +nbclient, IPython, numpy, matplotlib, TensorBoard and a Python 3 Jupyter kernel. +Jupyter needs local socket permissions. Chrome/Chromium is optional for HTML +but required for PDF. `MPLCONFIGDIR` defaults to a temporary writable cache; +scientific inputs and outputs remain study-relative. + +`inputs//` contains complete histories, complete +`plots_step_50000`, the rank-zero `train.log.0` copied as `train.log`, separate +nonempty `tensorboard/train` and `tensorboard/valid` streams, train configuration, +hyperparameters, scenario manifest and resolved particleflow specification. +Files are byte-for-byte copies. `inputs/run_manifest.json` maps descriptive names +to exact original run IDs. `inputs/SHA256SUMS` checks every archived input. +`inputs/sources/` contains the campaign revision's metric, data-sampling and +timing code for definition auditing. No checkpoint, prediction parquet, model +pickle or dataset cache was copied. Inputs total approximately 428.6 MB, mainly +TensorBoard. There is no dependency on a previous study's local artifacts. + +New study files follow the parent directory's local-artifact/Git-ignore policy. +Preserve the full dated folder to retain reproducibility. No private-bucket +upload was requested or performed. diff --git a/clic/clusters/v3.2.0/comparison_metrics/summary.json b/clic/clusters/v3.2.0/comparison_metrics/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5dbd7df57c5a25c2ec53c0854c70285ac8280cdf --- /dev/null +++ b/clic/clusters/v3.2.0/comparison_metrics/summary.json @@ -0,0 +1,44 @@ +[ + { + "run": "clic_pf_attention", + "detector": "CLIC", + "model": "Track/cluster attention", + "f1": 0.8389230370521545, + "efficiency": 0.8360208868980408, + "pt_mae": 0.07398535311222076, + "jet_iqr": 0.06730797135880051, + "jet_recall": 0.9448917749635634, + "met": 4.248844146728516, + "p3": 12.809347152709961, + "steady_s": 0.1119771409034729, + "devices": 1, + "params": 3114258, + "wait_ms": 2.0476579666137695, + "wait_pct": 0.15171306378560595, + "transfer_ms": 1.2761279940605164, + "compute_s": 0.2169132666895166, + "sampled_s": 1.3458307981491089, + "peak_gib": 2.252988338470459 + }, + { + "run": "clic_pf_heptv2", + "detector": "CLIC", + "model": "Track/cluster HEPTv2", + "f1": 0.8450756072998047, + "efficiency": 0.8401479125022888, + "pt_mae": 0.0721035972237587, + "jet_iqr": 0.0604938331773314, + "jet_recall": 0.9476291197222206, + "met": 3.8686211109161377, + "p3": 11.288646697998047, + "steady_s": 0.4003996539115906, + "devices": 1, + "params": 8639954, + "wait_ms": 1.985311508178711, + "wait_pct": 0.12339891821795848, + "transfer_ms": 1.2515810085460544, + "compute_s": 0.5076365884160623, + "sampled_s": 1.6224647760391235, + "peak_gib": 3.030461311340332 + } +] diff --git a/clic/clusters/v3.2.0/release.json b/clic/clusters/v3.2.0/release.json new file mode 100644 index 0000000000000000000000000000000000000000..819b202bc69572558e9f2a0b1dce2f3ea0b6d088 --- /dev/null +++ b/clic/clusters/v3.2.0/release.json @@ -0,0 +1,72 @@ +{ + "release": "3.2.0", + "detector": "clic", + "representation": "clusters", + "training_code_revision": "6b80b983477321cf708e5043fb1d2ec15dd398bc", + "selected_step": 50000, + "dataset_version": "3.3.0", + "models": { + "attention": { + "run": "clic_pf_attention_seed12345_20261028_111202_052283", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "a87d438bbd6432178ad72c3e86e4031244fb6baf8c2044a2c3843fcc78a575a8", + "parameters": 3114258, + "global_batch_size": 128, + "accelerator_devices": 1, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8389230370521545, + "metrics/particle/matched/pt_relative_abs_mean": 0.07398535311222076, + "metrics/particle/event/met_abs_error": 4.248844146728516, + "metrics/particle/momentum/all/event/p3_error_mean": 12.809347152709961 + } + }, + "heptv2": { + "run": "clic_pf_heptv2_seed12345_20261028_111202_849554", + "checkpoint": "checkpoints/checkpoint-50000.pth", + "checkpoint_sha256": "4ad2caa8fcc5adf4a316b44231e6bddbf83606b701a566b8f3f57876b756ab2e", + "parameters": 8639954, + "global_batch_size": 128, + "accelerator_devices": 1, + "seed": 12345, + "history_points": 50, + "missing_history_steps": [], + "validation_metrics": { + "metrics/particle/matching/f1": 0.8450756072998047, + "metrics/particle/matched/pt_relative_abs_mean": 0.0721035972237587, + "metrics/particle/event/met_abs_error": 3.8686211109161377, + "metrics/particle/momentum/all/event/p3_error_mean": 11.288646697998047 + } + } + }, + "contents": "Final 50k checkpoints, complete available histories, separate train/valid TensorBoard streams, configs, model kwargs, scenario manifests, all training logs, and complete final plots. Prediction parquet files and batch dumps are excluded.", + "checkpoint_selection": "Final scheduled 50k checkpoint; not selected as best by validation or test performance.", + "code_tag": "v3.2.0 is the intended code release label. This upload does not create or verify the Git tag. The training commit above is authoritative.", + "limitations": [ + "One seed. No paired confidence intervals or verified cross-representation event identity.", + "PF TFDS 3.3.0 versus hits 3.2.2, with differing target populations and validation sampling.", + "PF validation has 1530 entries, distributed hit validation pads to 1536; detailed momentum diagnostics use differently selected 512-entry subsets.", + "Training resource figures are not inference or end-to-end reconstruction benchmarks.", + "CLD hit HEPTv2 has no step_30000.json in the source results." + ], + "checkpoint_validation": [ + { + "run": "clic_pf_attention_seed12345_20261028_111202_052283", + "step": 50000, + "parameters": 3114258, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + }, + { + "run": "clic_pf_heptv2_seed12345_20261028_111202_849554", + "step": 50000, + "parameters": 8639954, + "strict_state_dict_load": true, + "validation_code_revision": "d2d991cea54ab47382e9de63bc900d2850e75eb4", + "scope": "CPU construction and strict state-dict loading including task-loss calibration buffers. No forward/inference or physics validation." + } + ] +}